Trace cbs which are accelerated without rnp lock help in rcu_accelerate_cbs_unlocked(). Signed-off-by: Neeraj Upadhyay <neeraju@xxxxxxxxxxxxxx> --- kernel/rcu/tree.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 338b817..097990a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1473,6 +1473,17 @@ static void rcu_gp_kthread_wake(void) swake_up_one(&rcu_state.gp_wq); } +static void rcu_trace_accelerated_cbs(struct rcu_data *rdp, + unsigned long gp_seq_req) +{ + rcu_lockdep_assert_cblist_protected(rdp); + /* Trace depending on how much we were able to accelerate. */ + if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) + trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); + else + trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); +} + /* * If there is room, assign a ->gp_seq number to any callbacks on this * CPU that have not already been assigned. Also accelerate any callbacks @@ -1511,12 +1522,7 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) if (!rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) return ret; ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); - - /* Trace depending on how much we were able to accelerate. */ - if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); - else - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); + rcu_trace_accelerated_cbs(rdp, gp_seq_req); return ret; } @@ -1538,7 +1544,8 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, c = rcu_seq_snap(&rcu_state.gp_seq); if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { /* Old request still live, so mark recent callbacks. */ - (void)rcu_segcblist_accelerate(&rdp->cblist, c); + if (rcu_segcblist_accelerate(&rdp->cblist, c)) + rcu_trace_accelerated_cbs(rdp, c); return; } raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ -- The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project