summaryrefslogtreecommitdiffstats
path: root/kernel/rcu
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.ibm.com>2019-06-26 09:50:38 -0700
committerPaul E. McKenney <paulmck@linux.ibm.com>2019-08-13 14:35:49 -0700
commitfaca5c250935262f026cac1bb951a0f7672474b8 (patch)
tree189d5640bf90a793916b66b0f77223f0411d8eca /kernel/rcu
parent4fd8c5f153bc41ae847b9ddb1539b34f70c18278 (diff)
downloadlinux-faca5c250935262f026cac1bb951a0f7672474b8.tar.bz2
rcu/nocb: Unconditionally advance and wake for excessive CBs
When there are excessive numbers of callbacks, and when either the corresponding no-CBs callback kthread is asleep or there is no more ready-to-invoke callbacks, and when least one callback is pending, __call_rcu_nocb_wake() will advance the callbacks, but refrain from awakening the corresponding no-CBs grace-period kthread. However, because rcu_advance_cbs_nowake() is used, it is possible (if a bit unlikely) that the needed advancement could not happen due to a grace period not being in progress. Plus there will always be at least one pending callback due to one having just now been enqueued. This commit therefore attempts to advance callbacks and awakens the no-CBs grace-period kthread when there are excessive numbers of callbacks posted and when the no-CBs callback kthread is not in a position to do anything helpful. Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
Diffstat (limited to 'kernel/rcu')
-rw-r--r--kernel/rcu/tree_plugin.h16
1 files changed, 11 insertions, 5 deletions
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index e92bc39c4008..4e49bb359464 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1668,13 +1668,19 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
if (!rdp->nocb_cb_sleep &&
rcu_segcblist_ready_cbs(&rdp->cblist)) {
// Already going full tilt, so don't try to rewake.
- } else if (rcu_segcblist_pend_cbs(&rdp->cblist)) {
- rcu_advance_cbs_nowake(rdp->mynode, rdp);
+ rcu_nocb_unlock_irqrestore(rdp, flags);
} else {
- wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
- TPS("WakeOvfIsDeferred"));
+ rcu_advance_cbs_nowake(rdp->mynode, rdp);
+ if (!irqs_disabled_flags(flags)) {
+ wake_nocb_gp(rdp, false, flags);
+ trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
+ TPS("WakeOvf"));
+ } else {
+ wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
+ TPS("WakeOvfIsDeferred"));
+ rcu_nocb_unlock_irqrestore(rdp, flags);
+ }
}
- rcu_nocb_unlock_irqrestore(rdp, flags);
} else {
trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
rcu_nocb_unlock_irqrestore(rdp, flags);