summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.ibm.com>2019-05-28 05:54:26 -0700
committerPaul E. McKenney <paulmck@linux.ibm.com>2019-08-13 14:35:49 -0700
commit7f36ef82e5cf0b401c2676fb3e56ad0633ed6ad5 (patch)
tree4c71dcb767da7b42feae5f1d159d2a604d829d03
parentce0a825e40606d6dbe6dfe90d4d4c0ccc9fa3bde (diff)
downloadlinux-7f36ef82e5cf0b401c2676fb3e56ad0633ed6ad5.tar.bz2
rcu/nocb: Avoid needless wakeups of no-CBs grace-period kthread
Currently, the code provides an extra wakeup for the no-CBs grace-period kthread if one of its CPUs is generating excessive numbers of callbacks. But satisfying though it is to wake something up when things are going south, unless the thing being awakened can actually help solve the problem, that extra wakeup does nothing but consume additional CPU time, which is exactly what you don't want during a call_rcu() flood. This commit therefore avoids doing anything if the corresponding no-CBs callback kthread is going full tilt. Otherwise, if advancing callbacks immediately might help and if the leaf rcu_node structure's lock is immediately available, this commit invokes a new variant of rcu_advance_cbs() that advances callbacks only if doing so won't require awakening the grace-period kthread (not to be confused with any of the no-CBs grace-period kthreads). Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
-rw-r--r--kernel/rcu/tree.c15
-rw-r--r--kernel/rcu/tree_plugin.h13
2 files changed, 24 insertions, 4 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index fb6b80aa34f6..a6ddfae6978d 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1335,6 +1335,19 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
}
/*
+ * Move and classify callbacks, but only if doing so won't require
+ * that the RCU grace-period kthread be awakened.
+ */
+static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
+ struct rcu_data *rdp)
+{
+ raw_lockdep_assert_held_rcu_node(rnp);
+ if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))
+ return;
+ WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
+}
+
+/*
* Update CPU-local rcu_data state to record the beginnings and ends of
* grace periods. The caller must hold the ->lock of the leaf rcu_node
* structure corresponding to the current CPU, and must have irqs disabled.
@@ -2118,6 +2131,8 @@ static void rcu_do_batch(struct rcu_data *rdp)
rcu_segcblist_n_lazy_cbs(&rdp->cblist),
rcu_segcblist_n_cbs(&rdp->cblist), bl);
rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
+ if (offloaded)
+ rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
rcu_nocb_unlock_irqrestore(rdp, flags);
/* Invoke callbacks. */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 5cbc0848647c..c10afe778430 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1643,10 +1643,15 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
} else if (len > rdp->qlen_last_fqs_check + qhimark) {
/* ... or if many callbacks queued. */
rdp->qlen_last_fqs_check = len;
- if (!irqs_disabled_flags(flags)) {
- wake_nocb_gp(rdp, true, flags);
- trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
- TPS("WakeOvf"));
+ if (!rdp->nocb_cb_sleep &&
+ rcu_segcblist_ready_cbs(&rdp->cblist)) {
+ // Already going full tilt, so don't try to rewake.
+ rcu_nocb_unlock_irqrestore(rdp, flags);
+ } else if (rcu_segcblist_pend_cbs(&rdp->cblist) &&
+ raw_spin_trylock_rcu_node(rdp->mynode)) {
+ rcu_advance_cbs_nowake(rdp->mynode, rdp);
+ raw_spin_unlock_rcu_node(rdp->mynode);
+ rcu_nocb_unlock_irqrestore(rdp, flags);
} else {
wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
TPS("WakeOvfIsDeferred"));