Trace cbs which are accelerated without rnp lock help in
rcu_accelerate_cbs_unlocked().
Signed-off-by: Neeraj Upadhyay <neer...@codeaurora.org>
---
kernel/rcu/tree.c | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 338b817..097990a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1473,6 +1473,17 @@ static void rcu_gp_kthread_wake(void)
swake_up_one(&rcu_state.gp_wq);
}
+static void rcu_trace_accelerated_cbs(struct rcu_data *rdp,
+ unsigned long gp_seq_req)
+{
+ rcu_lockdep_assert_cblist_protected(rdp);
+ /* Trace depending on how much we were able to accelerate. */
+ if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
+ trace_rcu_grace_period(rcu_state.name, gp_seq_req,
TPS("AccWaitCB"));
+ else
+ trace_rcu_grace_period(rcu_state.name, gp_seq_req,
TPS("AccReadyCB"));
+}
+
/*
* If there is room, assign a ->gp_seq number to any callbacks on this
* CPU that have not already been assigned. Also accelerate any callbacks
@@ -1511,12 +1522,7 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp,
struct rcu_data *rdp)
if (!rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
return ret;
ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
-
- /* Trace depending on how much we were able to accelerate. */
- if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
- trace_rcu_grace_period(rcu_state.name, gp_seq_req,
TPS("AccWaitCB"));
- else
- trace_rcu_grace_period(rcu_state.name, gp_seq_req,
TPS("AccReadyCB"));
+ rcu_trace_accelerated_cbs(rdp, gp_seq_req);
return ret;
}
@@ -1538,7 +1544,8 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node
*rnp,
c = rcu_seq_snap(&rcu_state.gp_seq);
if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
/* Old request still live, so mark recent callbacks. */
- (void)rcu_segcblist_accelerate(&rdp->cblist, c);
+ if (rcu_segcblist_accelerate(&rdp->cblist, c))
+ rcu_trace_accelerated_cbs(rdp, c);
return;
}
raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
--
The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
a Linux Foundation Collaborative Project
