summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--kernel/rcutree.c41
-rw-r--r--kernel/rcutree.h3
2 files changed, 36 insertions, 8 deletions
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 79fa2db1595b..d62c04482228 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -366,11 +366,9 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
*/
static void rcu_eqs_enter(bool user)
{
- unsigned long flags;
long long oldval;
struct rcu_dynticks *rdtp;
- local_irq_save(flags);
rdtp = &__get_cpu_var(rcu_dynticks);
oldval = rdtp->dynticks_nesting;
WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
@@ -379,7 +377,6 @@ static void rcu_eqs_enter(bool user)
else
rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;
rcu_eqs_enter_common(rdtp, oldval, user);
- local_irq_restore(flags);
}
/**
@@ -396,7 +393,11 @@ static void rcu_eqs_enter(bool user)
*/
void rcu_idle_enter(void)
{
+ unsigned long flags;
+
+ local_irq_save(flags);
rcu_eqs_enter(0);
+ local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(rcu_idle_enter);
@@ -411,6 +412,9 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter);
*/
void rcu_user_enter(void)
{
+ unsigned long flags;
+ struct rcu_dynticks *rdtp;
+
/*
* Some contexts may involve an exception occuring in an irq,
* leading to that nesting:
@@ -422,7 +426,15 @@ void rcu_user_enter(void)
if (in_interrupt())
return;
- rcu_eqs_enter(1);
+ WARN_ON_ONCE(!current->mm);
+
+ local_irq_save(flags);
+ rdtp = &__get_cpu_var(rcu_dynticks);
+ if (!rdtp->in_user) {
+ rdtp->in_user = true;
+ rcu_eqs_enter(1);
+ }
+ local_irq_restore(flags);
}
/**
@@ -516,11 +528,9 @@ static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
*/
static void rcu_eqs_exit(bool user)
{
- unsigned long flags;
struct rcu_dynticks *rdtp;
long long oldval;
- local_irq_save(flags);
rdtp = &__get_cpu_var(rcu_dynticks);
oldval = rdtp->dynticks_nesting;
WARN_ON_ONCE(oldval < 0);
@@ -529,7 +539,6 @@ static void rcu_eqs_exit(bool user)
else
rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
rcu_eqs_exit_common(rdtp, oldval, user);
- local_irq_restore(flags);
}
/**
@@ -545,7 +554,11 @@ static void rcu_eqs_exit(bool user)
*/
void rcu_idle_exit(void)
{
+ unsigned long flags;
+
+ local_irq_save(flags);
rcu_eqs_exit(0);
+ local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(rcu_idle_exit);
@@ -558,6 +571,9 @@ EXPORT_SYMBOL_GPL(rcu_idle_exit);
*/
void rcu_user_exit(void)
{
+ unsigned long flags;
+ struct rcu_dynticks *rdtp;
+
/*
* Some contexts may involve an exception occuring in an irq,
* leading to that nesting:
@@ -569,7 +585,13 @@ void rcu_user_exit(void)
if (in_interrupt())
return;
- rcu_eqs_exit(1);
+ local_irq_save(flags);
+ rdtp = &__get_cpu_var(rcu_dynticks);
+ if (rdtp->in_user) {
+ rdtp->in_user = false;
+ rcu_eqs_exit(1);
+ }
+ local_irq_restore(flags);
}
/**
@@ -2586,6 +2608,9 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
+#ifdef CONFIG_RCU_USER_QS
+ WARN_ON_ONCE(rdp->dynticks->in_user);
+#endif
rdp->cpu = cpu;
rdp->rsp = rsp;
raw_spin_unlock_irqrestore(&rnp->lock, flags);
diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index 7576fd4d8ce6..10cc2f9f8433 100644
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -102,6 +102,9 @@ struct rcu_dynticks {
/* idle-period nonlazy_posted snapshot. */
int tick_nohz_enabled_snap; /* Previously seen value from sysfs. */
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
+#ifdef CONFIG_RCU_USER_QS
+ bool in_user; /* Is the CPU in userland from RCU POV? */
+#endif
};
/* RCU's kthread states for tracing. */