From 0cd7e350abc40eed5d3b60292dc102f700c88388 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 22 Nov 2022 13:53:57 -0800 Subject: rcu: Make SRCU mandatory Kernels configured with CONFIG_PRINTK=n and CONFIG_SRCU=n get build failures. This causes trouble for deep embedded systems. But given that there are more than 25 instances of "select SRCU" in the kernel, it is hard to believe that there are many kernels running in production without SRCU. This commit therefore makes SRCU mandatory. The SRCU Kconfig option remains for backwards compatibility, and will be removed when it is no longer used. [ paulmck: Update per kernel test robot feedback. ] Reported-by: John Ogness Reported-by: Petr Mladek Signed-off-by: Paul E. McKenney Cc: Acked-by: Randy Dunlap # build-tested Reviewed-by: John Ogness --- include/linux/rcutiny.h | 4 ---- include/linux/srcu.h | 9 +-------- 2 files changed, 1 insertion(+), 12 deletions(-) (limited to 'include') diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 768196a5f39d..d40b21ec7e0d 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -154,11 +154,7 @@ static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) return false; } static inline void rcu_preempt_deferred_qs(struct task_struct *t) { } -#ifdef CONFIG_SRCU void rcu_scheduler_starting(void); -#else /* #ifndef CONFIG_SRCU */ -static inline void rcu_scheduler_starting(void) { } -#endif /* #else #ifndef CONFIG_SRCU */ static inline void rcu_end_inkernel_boot(void) { } static inline bool rcu_inkernel_boot_has_ended(void) { return true; } static inline bool rcu_is_watching(void) { return true; } diff --git a/include/linux/srcu.h b/include/linux/srcu.h index f0814ffca34b..9b9d0bbf1d3c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -47,11 +47,8 @@ int init_srcu_struct(struct srcu_struct *ssp); #include #elif defined(CONFIG_TREE_SRCU) #include -#elif defined(CONFIG_SRCU) -#error "Unknown SRCU implementation specified to kernel configuration" #else -/* Dummy definition for things like notifiers. Actual use gets link error. */ -struct srcu_struct { }; +#error "Unknown SRCU implementation specified to kernel configuration" #endif void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, @@ -78,11 +75,7 @@ static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) } #endif /* CONFIG_NEED_SRCU_NMI_SAFE */ -#ifdef CONFIG_SRCU void srcu_init(void); -#else /* #ifdef CONFIG_SRCU */ -static inline void srcu_init(void) { } -#endif /* #else #ifdef CONFIG_SRCU */ #ifdef CONFIG_DEBUG_LOCK_ALLOC -- cgit v1.2.3 From d9a4af5690e26afa8a2eb83c575d3a9ef52cde1d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 16 Nov 2022 17:27:14 +0106 Subject: printk: Convert console_drivers list to hlist Replace the open coded single linked list with a hlist so a conversion to SRCU protected list walks can reuse the existing primitives. Co-developed-by: John Ogness Signed-off-by: John Ogness Signed-off-by: Thomas Gleixner Reviewed-by: Greg Kroah-Hartman Reviewed-by: Sergey Senozhatsky Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20221116162152.193147-3-john.ogness@linutronix.de --- fs/proc/consoles.c | 3 +- include/linux/console.h | 8 ++-- kernel/printk/printk.c | 101 ++++++++++++++++++++++++++---------------------- 3 files changed, 62 insertions(+), 50 deletions(-) (limited to 'include') diff --git a/fs/proc/consoles.c b/fs/proc/consoles.c index dfe6ce3505ce..cf2e0788f9c7 100644 --- a/fs/proc/consoles.c +++ b/fs/proc/consoles.c @@ -74,8 +74,9 @@ static void *c_start(struct seq_file *m, loff_t *pos) static void *c_next(struct seq_file *m, void *v, loff_t *pos) { struct console *con = v; + ++*pos; - return con->next; + return hlist_entry_safe(con->node.next, struct console, node); } static void c_stop(struct seq_file *m, void *v) diff --git a/include/linux/console.h b/include/linux/console.h index 8c1686e2c233..7b5f21f9e469 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -15,6 +15,7 @@ #define _LINUX_CONSOLE_H_ 1 #include +#include #include struct vc_data; @@ -154,14 +155,16 @@ struct console { u64 seq; unsigned long dropped; void *data; - struct console *next; + struct hlist_node node; }; +extern struct hlist_head console_list; + /* * for_each_console() allows you to iterate on each console */ #define for_each_console(con) \ - for (con = console_drivers; con != NULL; con = con->next) + hlist_for_each_entry(con, &console_list, node) extern int console_set_on_cmdline; extern struct console *early_console; @@ -174,7 +177,6 @@ enum con_flush_mode { extern int add_preferred_console(char *name, int idx, char *options); extern void register_console(struct console *); extern int unregister_console(struct console *); -extern struct console *console_drivers; extern void console_lock(void); extern int console_trylock(void); extern void console_unlock(void); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index e4f1e7478b52..e6f0832e71f0 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -79,13 +79,12 @@ int oops_in_progress; EXPORT_SYMBOL(oops_in_progress); /* - * console_sem protects the console_drivers list, and also - * provides serialisation for access to the entire console - * driver system. + * console_sem protects console_list and console->flags updates, and also + * provides serialization for access to the entire console driver system. */ static DEFINE_SEMAPHORE(console_sem); -struct console *console_drivers; -EXPORT_SYMBOL_GPL(console_drivers); +HLIST_HEAD(console_list); +EXPORT_SYMBOL_GPL(console_list); /* * System may need to suppress printk message under certain @@ -2556,7 +2555,7 @@ static int console_cpu_notify(unsigned int cpu) * console_lock - lock the console system for exclusive use. * * Acquires a lock which guarantees that the caller has - * exclusive access to the console system and the console_drivers list. + * exclusive access to the console system and console_list. * * Can sleep, returns nothing. */ @@ -2576,7 +2575,7 @@ EXPORT_SYMBOL(console_lock); * console_trylock - try to lock the console system for exclusive use. * * Try to acquire a lock which guarantees that the caller has exclusive - * access to the console system and the console_drivers list. + * access to the console system and console_list. * * returns 1 on success, and 0 on failure to acquire the lock. */ @@ -2940,11 +2939,20 @@ void console_flush_on_panic(enum con_flush_mode mode) console_may_schedule = 0; if (mode == CONSOLE_REPLAY_ALL) { + struct hlist_node *tmp; struct console *c; u64 seq; seq = prb_first_valid_seq(prb); - for_each_console(c) + /* + * This cannot use for_each_console() because it's not established + * that the current context has console locked and neither there is + * a guarantee that there is no concurrency in that case. + * + * Open code it for documentation purposes and pretend that + * it works. + */ + hlist_for_each_entry_safe(c, tmp, &console_list, node) c->seq = seq; } console_unlock(); @@ -3081,6 +3089,9 @@ static void try_enable_default_console(struct console *newcon) (con->flags & CON_BOOT) ? "boot" : "", \ con->name, con->index, ##__VA_ARGS__) +#define console_first() \ + hlist_entry(console_list.first, struct console, node) + /* * The console driver calls this routine during kernel initialization * to register the console printing procedure with printk() and to @@ -3140,8 +3151,8 @@ void register_console(struct console *newcon) * flag set and will be first in the list. */ if (preferred_console < 0) { - if (!console_drivers || !console_drivers->device || - console_drivers->flags & CON_BOOT) { + if (hlist_empty(&console_list) || !console_first()->device || + console_first()->flags & CON_BOOT) { try_enable_default_console(newcon); } } @@ -3169,20 +3180,22 @@ void register_console(struct console *newcon) } /* - * Put this console in the list - keep the - * preferred driver at the head of the list. + * Put this console in the list - keep the + * preferred driver at the head of the list. */ console_lock(); - if ((newcon->flags & CON_CONSDEV) || console_drivers == NULL) { - newcon->next = console_drivers; - console_drivers = newcon; - if (newcon->next) - newcon->next->flags &= ~CON_CONSDEV; - /* Ensure this flag is always set for the head of the list */ + if (hlist_empty(&console_list)) { + /* Ensure CON_CONSDEV is always set for the head. */ newcon->flags |= CON_CONSDEV; + hlist_add_head(&newcon->node, &console_list); + + } else if (newcon->flags & CON_CONSDEV) { + /* Only the new head can have CON_CONSDEV set. */ + console_first()->flags &= ~CON_CONSDEV; + hlist_add_head(&newcon->node, &console_list); + } else { - newcon->next = console_drivers->next; - console_drivers->next = newcon; + hlist_add_behind(&newcon->node, console_list.first); } newcon->dropped = 0; @@ -3209,16 +3222,18 @@ void register_console(struct console *newcon) if (bootcon_enabled && ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) && !keep_bootcon) { - for_each_console(con) + struct hlist_node *tmp; + + hlist_for_each_entry_safe(con, tmp, &console_list, node) { if (con->flags & CON_BOOT) unregister_console(con); + } } } EXPORT_SYMBOL(register_console); int unregister_console(struct console *console) { - struct console *con; int res; con_printk(KERN_INFO, console, "disabled\n"); @@ -3229,32 +3244,30 @@ int unregister_console(struct console *console) if (res > 0) return 0; - res = -ENODEV; console_lock(); - if (console_drivers == console) { - console_drivers=console->next; - res = 0; - } else { - for_each_console(con) { - if (con->next == console) { - con->next = console->next; - res = 0; - break; - } - } + + /* Disable it unconditionally */ + console->flags &= ~CON_ENABLED; + + if (hlist_unhashed(&console->node)) { + console_unlock(); + return -ENODEV; } - if (res) - goto out_disable_unlock; + hlist_del_init(&console->node); /* + * * If this isn't the last console and it has CON_CONSDEV set, we * need to set it on the next preferred console. + * + * + * The above makes no sense as there is no guarantee that the next + * console has any device attached. Oh well.... */ - if (console_drivers != NULL && console->flags & CON_CONSDEV) - console_drivers->flags |= CON_CONSDEV; + if (!hlist_empty(&console_list) && console->flags & CON_CONSDEV) + console_first()->flags |= CON_CONSDEV; - console->flags &= ~CON_ENABLED; console_unlock(); console_sysfs_notify(); @@ -3262,12 +3275,6 @@ int unregister_console(struct console *console) res = console->exit(console); return res; - -out_disable_unlock: - console->flags &= ~CON_ENABLED; - console_unlock(); - - return res; } EXPORT_SYMBOL(unregister_console); @@ -3317,10 +3324,11 @@ void __init console_init(void) */ static int __init printk_late_init(void) { + struct hlist_node *tmp; struct console *con; int ret; - for_each_console(con) { + hlist_for_each_entry_safe(con, tmp, &console_list, node) { if (!(con->flags & CON_BOOT)) continue; @@ -3340,6 +3348,7 @@ static int __init printk_late_init(void) unregister_console(con); } } + ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL, console_cpu_notify); WARN_ON(ret < 0); -- cgit v1.2.3 From 6c4afa79147e4aa86665795695ac4a5f25e73176 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 16 Nov 2022 17:27:15 +0106 Subject: printk: Prepare for SRCU console list protection Provide an NMI-safe SRCU protected variant to walk the console list. Note that all console fields are now set before adding the console to the list to avoid the console becoming visible by SCRU readers before being fully initialized. This is a preparatory change for a new console infrastructure which operates independent of the console BKL. Suggested-by: Thomas Gleixner Signed-off-by: John Ogness Acked-by: Miguel Ojeda Reviewed-by: Paul E. McKenney Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20221116162152.193147-4-john.ogness@linutronix.de --- .clang-format | 1 + include/linux/console.h | 28 +++++++++++++++- kernel/printk/printk.c | 87 ++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 100 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/.clang-format b/.clang-format index 1247d54f9e49..04a675b56b57 100644 --- a/.clang-format +++ b/.clang-format @@ -222,6 +222,7 @@ ForEachMacros: - 'for_each_component_dais' - 'for_each_component_dais_safe' - 'for_each_console' + - 'for_each_console_srcu' - 'for_each_cpu' - 'for_each_cpu_and' - 'for_each_cpu_not' diff --git a/include/linux/console.h b/include/linux/console.h index 7b5f21f9e469..f4f0c9523835 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -15,7 +15,7 @@ #define _LINUX_CONSOLE_H_ 1 #include -#include +#include #include struct vc_data; @@ -158,8 +158,34 @@ struct console { struct hlist_node node; }; +#ifdef CONFIG_DEBUG_LOCK_ALLOC +extern bool console_srcu_read_lock_is_held(void); +#else +static inline bool console_srcu_read_lock_is_held(void) +{ + return 1; +} +#endif + +extern int console_srcu_read_lock(void); +extern void console_srcu_read_unlock(int cookie); + extern struct hlist_head console_list; +/** + * for_each_console_srcu() - Iterator over registered consoles + * @con: struct console pointer used as loop cursor + * + * Although SRCU guarantees the console list will be consistent, the + * struct console fields may be updated by other CPUs while iterating. + * + * Requires console_srcu_read_lock to be held. Can be invoked from + * any context. + */ +#define for_each_console_srcu(con) \ + hlist_for_each_entry_srcu(con, &console_list, node, \ + console_srcu_read_lock_is_held()) + /* * for_each_console() allows you to iterate on each console */ diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index e6f0832e71f0..173f46a29252 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -85,6 +85,7 @@ EXPORT_SYMBOL(oops_in_progress); static DEFINE_SEMAPHORE(console_sem); HLIST_HEAD(console_list); EXPORT_SYMBOL_GPL(console_list); +DEFINE_STATIC_SRCU(console_srcu); /* * System may need to suppress printk message under certain @@ -104,6 +105,13 @@ static struct lockdep_map console_lock_dep_map = { }; #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC +bool console_srcu_read_lock_is_held(void) +{ + return srcu_read_lock_held(&console_srcu); +} +#endif + enum devkmsg_log_bits { __DEVKMSG_LOG_BIT_ON = 0, __DEVKMSG_LOG_BIT_OFF, @@ -219,6 +227,32 @@ int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, } #endif /* CONFIG_PRINTK && CONFIG_SYSCTL */ +/** + * console_srcu_read_lock - Register a new reader for the + * SRCU-protected console list + * + * Use for_each_console_srcu() to iterate the console list + * + * Context: Any context. + */ +int console_srcu_read_lock(void) +{ + return srcu_read_lock_nmisafe(&console_srcu); +} +EXPORT_SYMBOL(console_srcu_read_lock); + +/** + * console_srcu_read_unlock - Unregister an old reader from + * the SRCU-protected console list + * + * Counterpart to console_srcu_read_lock() + */ +void console_srcu_read_unlock(int cookie) +{ + srcu_read_unlock_nmisafe(&console_srcu, cookie); +} +EXPORT_SYMBOL(console_srcu_read_unlock); + /* * Helper macros to handle lockdep when locking/unlocking console_sem. We use * macros instead of functions so that _RET_IP_ contains useful information. @@ -2989,6 +3023,14 @@ void console_stop(struct console *console) console_lock(); console->flags &= ~CON_ENABLED; console_unlock(); + + /* + * Ensure that all SRCU list walks have completed. All contexts must + * be able to see that this console is disabled so that (for example) + * the caller can suspend the port without risk of another context + * using the port. + */ + synchronize_srcu(&console_srcu); } EXPORT_SYMBOL(console_stop); @@ -3179,6 +3221,17 @@ void register_console(struct console *newcon) newcon->flags &= ~CON_PRINTBUFFER; } + newcon->dropped = 0; + if (newcon->flags & CON_PRINTBUFFER) { + /* Get a consistent copy of @syslog_seq. */ + mutex_lock(&syslog_lock); + newcon->seq = syslog_seq; + mutex_unlock(&syslog_lock); + } else { + /* Begin with next message. */ + newcon->seq = prb_next_seq(prb); + } + /* * Put this console in the list - keep the * preferred driver at the head of the list. @@ -3187,28 +3240,24 @@ void register_console(struct console *newcon) if (hlist_empty(&console_list)) { /* Ensure CON_CONSDEV is always set for the head. */ newcon->flags |= CON_CONSDEV; - hlist_add_head(&newcon->node, &console_list); + hlist_add_head_rcu(&newcon->node, &console_list); } else if (newcon->flags & CON_CONSDEV) { /* Only the new head can have CON_CONSDEV set. */ console_first()->flags &= ~CON_CONSDEV; - hlist_add_head(&newcon->node, &console_list); + hlist_add_head_rcu(&newcon->node, &console_list); } else { - hlist_add_behind(&newcon->node, console_list.first); - } - - newcon->dropped = 0; - if (newcon->flags & CON_PRINTBUFFER) { - /* Get a consistent copy of @syslog_seq. */ - mutex_lock(&syslog_lock); - newcon->seq = syslog_seq; - mutex_unlock(&syslog_lock); - } else { - /* Begin with next message. */ - newcon->seq = prb_next_seq(prb); + hlist_add_behind_rcu(&newcon->node, console_list.first); } console_unlock(); + + /* + * No need to synchronize SRCU here! The caller does not rely + * on all contexts being able to see the new console before + * register_console() completes. + */ + console_sysfs_notify(); /* @@ -3254,7 +3303,7 @@ int unregister_console(struct console *console) return -ENODEV; } - hlist_del_init(&console->node); + hlist_del_init_rcu(&console->node); /* * @@ -3269,6 +3318,14 @@ int unregister_console(struct console *console) console_first()->flags |= CON_CONSDEV; console_unlock(); + + /* + * Ensure that all SRCU list walks have completed. All contexts + * must not be able to see this console in the list so that any + * exit/cleanup routines can be performed safely. + */ + synchronize_srcu(&console_srcu); + console_sysfs_notify(); if (console->exit) -- cgit v1.2.3 From 4dc64682ad37abb02c54ca598430ac27de60c21a Mon Sep 17 00:00:00 2001 From: John Ogness Date: Mon, 21 Nov 2022 12:16:12 +0106 Subject: printk: introduce console_list_lock Currently there exist races in register_console(), where the types of registered consoles are checked (without holding the console_lock) and then after acquiring the console_lock, it is assumed that the list has not changed. Also, some code that performs console_unregister() make similar assumptions. It might be possible to fix these races using the console_lock. But it would require a complex analysis of all console drivers to make sure that the console_lock is not taken in match() and setup() callbacks. And we really prefer to split up and reduce the responsibilities of console_lock rather than expand its complexity. Therefore, introduce a new console_list_lock to provide full synchronization for any console list changes. In addition, also use console_list_lock for synchronization of console->flags updates. All flags are either static or modified only during the console registration. There are only two exceptions. The first exception is CON_ENABLED, which is also modified by console_start()/console_stop(). Therefore, these functions must also take the console_list_lock. The second exception is when the flags are modified by the console driver init code before the console is registered. These will be ignored because they are not visible to the rest of the system via the console_drivers list. Note that one of the various responsibilities of the console_lock is also intended to provide console list and console->flags synchronization. Later changes will update call sites relying on the console_lock for these purposes. Once all call sites have been updated, the console_lock will be relieved of synchronizing console_list and console->flags updates. Suggested-by: Thomas Gleixner Signed-off-by: John Ogness Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/87sficwokr.fsf@jogness.linutronix.de --- include/linux/console.h | 23 +++++++++++-- kernel/printk/printk.c | 89 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 100 insertions(+), 12 deletions(-) (limited to 'include') diff --git a/include/linux/console.h b/include/linux/console.h index f4f0c9523835..24d83e24840b 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -158,6 +158,14 @@ struct console { struct hlist_node node; }; +#ifdef CONFIG_LOCKDEP +extern void lockdep_assert_console_list_lock_held(void); +#else +static inline void lockdep_assert_console_list_lock_held(void) +{ +} +#endif + #ifdef CONFIG_DEBUG_LOCK_ALLOC extern bool console_srcu_read_lock_is_held(void); #else @@ -170,6 +178,9 @@ static inline bool console_srcu_read_lock_is_held(void) extern int console_srcu_read_lock(void); extern void console_srcu_read_unlock(int cookie); +extern void console_list_lock(void) __acquires(console_mutex); +extern void console_list_unlock(void) __releases(console_mutex); + extern struct hlist_head console_list; /** @@ -186,10 +197,16 @@ extern struct hlist_head console_list; hlist_for_each_entry_srcu(con, &console_list, node, \ console_srcu_read_lock_is_held()) -/* - * for_each_console() allows you to iterate on each console +/** + * for_each_console() - Iterator over registered consoles + * @con: struct console pointer used as loop cursor + * + * The console list and the console->flags are immutable while iterating. + * + * Requires console_list_lock to be held. */ -#define for_each_console(con) \ +#define for_each_console(con) \ + lockdep_assert_console_list_lock_held(); \ hlist_for_each_entry(con, &console_list, node) extern int console_set_on_cmdline; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c84654444a02..2b4506673a86 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -78,6 +78,13 @@ EXPORT_SYMBOL(ignore_console_lock_warning); int oops_in_progress; EXPORT_SYMBOL(oops_in_progress); +/* + * console_mutex protects console_list updates and console->flags updates. + * The flags are synchronized only for consoles that are registered, i.e. + * accessible via the console list. + */ +static DEFINE_MUTEX(console_mutex); + /* * console_sem protects console_list and console->flags updates, and also * provides serialization for access to the entire console driver system. @@ -103,6 +110,12 @@ static int __read_mostly suppress_panic_printk; static struct lockdep_map console_lock_dep_map = { .name = "console_lock" }; + +void lockdep_assert_console_list_lock_held(void) +{ + lockdep_assert_held(&console_mutex); +} +EXPORT_SYMBOL(lockdep_assert_console_list_lock_held); #endif #ifdef CONFIG_DEBUG_LOCK_ALLOC @@ -227,6 +240,40 @@ int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, } #endif /* CONFIG_PRINTK && CONFIG_SYSCTL */ +/** + * console_list_lock - Lock the console list + * + * For console list or console->flags updates + */ +void console_list_lock(void) +{ + /* + * In unregister_console(), synchronize_srcu() is called with the + * console_list_lock held. Therefore it is not allowed that the + * console_list_lock is taken with the srcu_lock held. + * + * Detecting if this context is really in the read-side critical + * section is only possible if the appropriate debug options are + * enabled. + */ + WARN_ON_ONCE(debug_lockdep_rcu_enabled() && + srcu_read_lock_held(&console_srcu)); + + mutex_lock(&console_mutex); +} +EXPORT_SYMBOL(console_list_lock); + +/** + * console_list_unlock - Unlock the console list + * + * Counterpart to console_list_lock() + */ +void console_list_unlock(void) +{ + mutex_unlock(&console_mutex); +} +EXPORT_SYMBOL(console_list_unlock); + /** * console_srcu_read_lock - Register a new reader for the * SRCU-protected console list @@ -3020,9 +3067,11 @@ struct tty_driver *console_device(int *index) void console_stop(struct console *console) { __pr_flush(console, 1000, true); + console_list_lock(); console_lock(); console->flags &= ~CON_ENABLED; console_unlock(); + console_list_unlock(); /* * Ensure that all SRCU list walks have completed. All contexts must @@ -3036,9 +3085,11 @@ EXPORT_SYMBOL(console_stop); void console_start(struct console *console) { + console_list_lock(); console_lock(); console->flags |= CON_ENABLED; console_unlock(); + console_list_unlock(); __pr_flush(console, 1000, true); } EXPORT_SYMBOL(console_start); @@ -3187,6 +3238,8 @@ static void console_init_seq(struct console *newcon, bool bootcon_registered) #define console_first() \ hlist_entry(console_list.first, struct console, node) +static int unregister_console_locked(struct console *console); + /* * The console driver calls this routine during kernel initialization * to register the console printing procedure with printk() and to @@ -3213,13 +3266,14 @@ void register_console(struct console *newcon) bool realcon_registered = false; int err; + console_list_lock(); + for_each_console(con) { if (WARN(con == newcon, "console '%s%d' already registered\n", - con->name, con->index)) - return; - } + con->name, con->index)) { + goto unlock; + } - for_each_console(con) { if (con->flags & CON_BOOT) bootcon_registered = true; else @@ -3230,7 +3284,7 @@ void register_console(struct console *newcon) if ((newcon->flags & CON_BOOT) && realcon_registered) { pr_info("Too late to register bootconsole %s%d\n", newcon->name, newcon->index); - return; + goto unlock; } /* @@ -3261,7 +3315,7 @@ void register_console(struct console *newcon) /* printk() messages are not printed to the Braille console. */ if (err || newcon->flags & CON_BRL) - return; + goto unlock; /* * If we have a bootconsole, and are switching to a real console, @@ -3320,16 +3374,21 @@ void register_console(struct console *newcon) hlist_for_each_entry_safe(con, tmp, &console_list, node) { if (con->flags & CON_BOOT) - unregister_console(con); + unregister_console_locked(con); } } +unlock: + console_list_unlock(); } EXPORT_SYMBOL(register_console); -int unregister_console(struct console *console) +/* Must be called under console_list_lock(). */ +static int unregister_console_locked(struct console *console) { int res; + lockdep_assert_console_list_lock_held(); + con_printk(KERN_INFO, console, "disabled\n"); res = _braille_unregister_console(console); @@ -3378,6 +3437,16 @@ int unregister_console(struct console *console) return res; } + +int unregister_console(struct console *console) +{ + int res; + + console_list_lock(); + res = unregister_console_locked(console); + console_list_unlock(); + return res; +} EXPORT_SYMBOL(unregister_console); /* @@ -3430,6 +3499,7 @@ static int __init printk_late_init(void) struct console *con; int ret; + console_list_lock(); hlist_for_each_entry_safe(con, tmp, &console_list, node) { if (!(con->flags & CON_BOOT)) continue; @@ -3447,9 +3517,10 @@ static int __init printk_late_init(void) */ pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n", con->name, con->index); - unregister_console(con); + unregister_console_locked(con); } } + console_list_unlock(); ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL, console_cpu_notify); -- cgit v1.2.3 From 100bdef2c198b8dc64df011dc4a2152db913c8ba Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 16 Nov 2022 17:27:24 +0106 Subject: console: introduce wrappers to read/write console flags After switching to SRCU for console list iteration, some readers will begin readings console->flags as a data race. Locklessly reading console->flags provides a consistent value because there is at most one CPU modifying console->flags and that CPU is using only read-modify-write operations. Introduce a wrapper for SRCU iterators to read console flags. Introduce a matching wrapper to write to flags of registered consoles. Writing to flags of registered consoles is synchronized by the console_list_lock. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20221116162152.193147-13-john.ogness@linutronix.de --- include/linux/console.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ kernel/printk/printk.c | 10 +++++----- 2 files changed, 50 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/include/linux/console.h b/include/linux/console.h index 24d83e24840b..c1ca461d088a 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -183,6 +183,51 @@ extern void console_list_unlock(void) __releases(console_mutex); extern struct hlist_head console_list; +/** + * console_srcu_read_flags - Locklessly read the console flags + * @con: struct console pointer of console to read flags from + * + * This function provides the necessary READ_ONCE() and data_race() + * notation for locklessly reading the console flags. The READ_ONCE() + * in this function matches the WRITE_ONCE() when @flags are modified + * for registered consoles with console_srcu_write_flags(). + * + * Only use this function to read console flags when locklessly + * iterating the console list via srcu. + * + * Context: Any context. + */ +static inline short console_srcu_read_flags(const struct console *con) +{ + WARN_ON_ONCE(!console_srcu_read_lock_is_held()); + + /* + * Locklessly reading console->flags provides a consistent + * read value because there is at most one CPU modifying + * console->flags and that CPU is using only read-modify-write + * operations to do so. + */ + return data_race(READ_ONCE(con->flags)); +} + +/** + * console_srcu_write_flags - Write flags for a registered console + * @con: struct console pointer of console to write flags to + * @flags: new flags value to write + * + * Only use this function to write flags for registered consoles. It + * requires holding the console_list_lock. + * + * Context: Any context. + */ +static inline void console_srcu_write_flags(struct console *con, short flags) +{ + lockdep_assert_console_list_lock_held(); + + /* This matches the READ_ONCE() in console_srcu_read_flags(). */ + WRITE_ONCE(con->flags, flags); +} + /** * for_each_console_srcu() - Iterator over registered consoles * @con: struct console pointer used as loop cursor diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 2b4506673a86..cad65d34c96d 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3069,7 +3069,7 @@ void console_stop(struct console *console) __pr_flush(console, 1000, true); console_list_lock(); console_lock(); - console->flags &= ~CON_ENABLED; + console_srcu_write_flags(console, console->flags & ~CON_ENABLED); console_unlock(); console_list_unlock(); @@ -3087,7 +3087,7 @@ void console_start(struct console *console) { console_list_lock(); console_lock(); - console->flags |= CON_ENABLED; + console_srcu_write_flags(console, console->flags | CON_ENABLED); console_unlock(); console_list_unlock(); __pr_flush(console, 1000, true); @@ -3343,7 +3343,7 @@ void register_console(struct console *newcon) } else if (newcon->flags & CON_CONSDEV) { /* Only the new head can have CON_CONSDEV set. */ - console_first()->flags &= ~CON_CONSDEV; + console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV); hlist_add_head_rcu(&newcon->node, &console_list); } else { @@ -3400,7 +3400,7 @@ static int unregister_console_locked(struct console *console) console_lock(); /* Disable it unconditionally */ - console->flags &= ~CON_ENABLED; + console_srcu_write_flags(console, console->flags & ~CON_ENABLED); if (hlist_unhashed(&console->node)) { console_unlock(); @@ -3419,7 +3419,7 @@ static int unregister_console_locked(struct console *console) * console has any device attached. Oh well.... */ if (!hlist_empty(&console_list) && console->flags & CON_CONSDEV) - console_first()->flags |= CON_CONSDEV; + console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV); console_unlock(); -- cgit v1.2.3 From 1fd4224a6b641f1949e27bf350b5b1c1e47e2ccc Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 16 Nov 2022 17:27:33 +0106 Subject: console: introduce console_is_registered() Currently it is not possible for drivers to detect if they have already successfully registered their console. Several drivers have multiple paths that lead to console registration. To avoid attempting a 2nd registration (which leads to a WARN), drivers are implementing their own solution. Introduce console_is_registered() so drivers can easily identify if their console is currently registered. A _locked() variant is also provided if the caller is already holding the console_list_lock. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20221116162152.193147-22-john.ogness@linutronix.de --- include/linux/console.h | 28 ++++++++++++++++++++++++++++ kernel/printk/printk.c | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/console.h b/include/linux/console.h index c1ca461d088a..f716e1dd9eaf 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -228,6 +228,34 @@ static inline void console_srcu_write_flags(struct console *con, short flags) WRITE_ONCE(con->flags, flags); } +/* Variant of console_is_registered() when the console_list_lock is held. */ +static inline bool console_is_registered_locked(const struct console *con) +{ + lockdep_assert_console_list_lock_held(); + return !hlist_unhashed(&con->node); +} + +/* + * console_is_registered - Check if the console is registered + * @con: struct console pointer of console to check + * + * Context: Process context. May sleep while acquiring console list lock. + * Return: true if the console is in the console list, otherwise false. + * + * If false is returned for a console that was previously registered, it + * can be assumed that the console's unregistration is fully completed, + * including the exit() callback after console list removal. + */ +static inline bool console_is_registered(const struct console *con) +{ + bool ret; + + console_list_lock(); + ret = console_is_registered_locked(con); + console_list_unlock(); + return ret; +} + /** * for_each_console_srcu() - Iterator over registered consoles * @con: struct console pointer used as loop cursor diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 101d08d9dc88..e58a6ccb945c 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3443,7 +3443,7 @@ static int unregister_console_locked(struct console *console) /* Disable it unconditionally */ console_srcu_write_flags(console, console->flags & ~CON_ENABLED); - if (hlist_unhashed(&console->node)) { + if (!console_is_registered_locked(console)) { console_unlock(); return -ENODEV; } -- cgit v1.2.3 From 452b9b24754044eced1508f9090611f3d9aa4ca5 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 16 Nov 2022 17:27:34 +0106 Subject: serial_core: replace uart_console_enabled() with uart_console_registered() All users of uart_console_enabled() really want to know if a console is registered. It is not reliable to check for CON_ENABLED in order to identify if a console is registered. Use console_is_registered() instead. A _locked() variant is provided because uart_set_options() is always called with the console_list_lock held and must check if a console is registered in order to synchronize with kgdboc. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20221116162152.193147-23-john.ogness@linutronix.de --- drivers/tty/serial/8250/8250_core.c | 2 +- drivers/tty/serial/pic32_uart.c | 2 +- drivers/tty/serial/serial_core.c | 14 +++++++------- include/linux/serial_core.h | 10 ++++++++-- 4 files changed, 17 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 94fbf0add2ce..74568292186f 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -565,7 +565,7 @@ serial8250_register_ports(struct uart_driver *drv, struct device *dev) up->port.dev = dev; - if (uart_console_enabled(&up->port)) + if (uart_console_registered(&up->port)) pm_runtime_get_sync(up->port.dev); serial8250_apply_quirks(up); diff --git a/drivers/tty/serial/pic32_uart.c b/drivers/tty/serial/pic32_uart.c index 2beada66c824..1183b2a26539 100644 --- a/drivers/tty/serial/pic32_uart.c +++ b/drivers/tty/serial/pic32_uart.c @@ -919,7 +919,7 @@ static int pic32_uart_probe(struct platform_device *pdev) } #ifdef CONFIG_SERIAL_PIC32_CONSOLE - if (uart_console_enabled(port)) { + if (uart_console_registered(port)) { /* The peripheral clock has been enabled by console_setup, * so disable it till the port is used. */ diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 179ee199df34..b9fbbee598b8 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -2223,11 +2223,11 @@ uart_set_options(struct uart_port *port, struct console *co, /* * Ensure that the serial-console lock is initialised early. * - * Note that the console-enabled check is needed because of kgdboc, - * which can end up calling uart_set_options() for an already enabled + * Note that the console-registered check is needed because + * kgdboc can call uart_set_options() for an already registered * console via tty_find_polling_driver() and uart_poll_init(). */ - if (!uart_console_enabled(port) && !port->console_reinit) + if (!uart_console_registered_locked(port) && !port->console_reinit) uart_port_spin_lock_init(port); memset(&termios, 0, sizeof(struct ktermios)); @@ -2573,7 +2573,7 @@ uart_configure_port(struct uart_driver *drv, struct uart_state *state, * successfully registered yet, try to re-register it. * It may be that the port was not available. */ - if (port->cons && !(port->cons->flags & CON_ENABLED)) + if (port->cons && !console_is_registered(port->cons)) register_console(port->cons); /* @@ -2956,7 +2956,7 @@ static ssize_t console_show(struct device *dev, mutex_lock(&port->mutex); uport = uart_port_check(state); if (uport) - console = uart_console_enabled(uport); + console = uart_console_registered(uport); mutex_unlock(&port->mutex); return sprintf(buf, "%c\n", console ? 'Y' : 'N'); @@ -2978,7 +2978,7 @@ static ssize_t console_store(struct device *dev, mutex_lock(&port->mutex); uport = uart_port_check(state); if (uport) { - oldconsole = uart_console_enabled(uport); + oldconsole = uart_console_registered(uport); if (oldconsole && !newconsole) { ret = unregister_console(uport->cons); } else if (!oldconsole && newconsole) { @@ -3086,7 +3086,7 @@ int uart_add_one_port(struct uart_driver *drv, struct uart_port *uport) * If this port is in use as a console then the spinlock is already * initialised. */ - if (!uart_console_enabled(uport)) + if (!uart_console_registered(uport)) uart_port_spin_lock_init(uport); if (uport->cons && uport->dev) diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index d657f2a42a7b..91871464b99d 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -743,9 +743,15 @@ static const bool earlycon_acpi_spcr_enable EARLYCON_USED_OR_UNUSED; static inline int setup_earlycon(char *buf) { return 0; } #endif -static inline bool uart_console_enabled(struct uart_port *port) +/* Variant of uart_console_registered() when the console_list_lock is held. */ +static inline bool uart_console_registered_locked(struct uart_port *port) { - return uart_console(port) && (port->cons->flags & CON_ENABLED); + return uart_console(port) && console_is_registered_locked(port->cons); +} + +static inline bool uart_console_registered(struct uart_port *port) +{ + return uart_console(port) && console_is_registered(port->cons); } struct uart_port *uart_get_console(struct uart_port *ports, int nr, -- cgit v1.2.3 From 6f8836756f3cbb02bfd1f0e033516585250318a9 Mon Sep 17 00:00:00 2001 From: John Ogness Date: Wed, 16 Nov 2022 17:27:44 +0106 Subject: printk, xen: fbfront: create/use safe function for forcing preferred With commit 9e124fe16ff2("xen: Enable console tty by default in domU if it's not a dummy") a hack was implemented to make sure that the tty console remains the console behind the /dev/console device. The main problem with the hack is that, after getting the console pointer to the tty console, it is assumed the pointer is still valid after releasing the console_sem. This assumption is incorrect and unsafe. Make the hack safe by introducing a new function console_force_preferred_locked() and perform the full operation under the console_list_lock. Signed-off-by: John Ogness Reviewed-by: Petr Mladek Signed-off-by: Petr Mladek Link: https://lore.kernel.org/r/20221116162152.193147-33-john.ogness@linutronix.de --- drivers/video/fbdev/xen-fbfront.c | 12 ++++------ include/linux/console.h | 1 + kernel/printk/printk.c | 49 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/drivers/video/fbdev/xen-fbfront.c b/drivers/video/fbdev/xen-fbfront.c index 4d2694d904aa..8752d389e382 100644 --- a/drivers/video/fbdev/xen-fbfront.c +++ b/drivers/video/fbdev/xen-fbfront.c @@ -504,18 +504,14 @@ static void xenfb_make_preferred_console(void) if (console_set_on_cmdline) return; - console_lock(); + console_list_lock(); for_each_console(c) { if (!strcmp(c->name, "tty") && c->index == 0) break; } - console_unlock(); - if (c) { - unregister_console(c); - c->flags |= CON_CONSDEV; - c->flags &= ~CON_PRINTBUFFER; /* don't print again */ - register_console(c); - } + if (c) + console_force_preferred_locked(c); + console_list_unlock(); } static int xenfb_resume(struct xenbus_device *dev) diff --git a/include/linux/console.h b/include/linux/console.h index f716e1dd9eaf..9cea254b34b8 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -291,6 +291,7 @@ enum con_flush_mode { }; extern int add_preferred_console(char *name, int idx, char *options); +extern void console_force_preferred_locked(struct console *con); extern void register_console(struct console *); extern int unregister_console(struct console *); extern void console_lock(void); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index e58a6ccb945c..bda9e8ec6817 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -248,9 +248,10 @@ int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, void console_list_lock(void) { /* - * In unregister_console(), synchronize_srcu() is called with the - * console_list_lock held. Therefore it is not allowed that the - * console_list_lock is taken with the srcu_lock held. + * In unregister_console() and console_force_preferred_locked(), + * synchronize_srcu() is called with the console_list_lock held. + * Therefore it is not allowed that the console_list_lock is taken + * with the srcu_lock held. * * Detecting if this context is really in the read-side critical * section is only possible if the appropriate debug options are @@ -3490,6 +3491,48 @@ int unregister_console(struct console *console) } EXPORT_SYMBOL(unregister_console); +/** + * console_force_preferred_locked - force a registered console preferred + * @con: The registered console to force preferred. + * + * Must be called under console_list_lock(). + */ +void console_force_preferred_locked(struct console *con) +{ + struct console *cur_pref_con; + + if (!console_is_registered_locked(con)) + return; + + cur_pref_con = console_first(); + + /* Already preferred? */ + if (cur_pref_con == con) + return; + + /* + * Delete, but do not re-initialize the entry. This allows the console + * to continue to appear registered (via any hlist_unhashed_lockless() + * checks), even though it was briefly removed from the console list. + */ + hlist_del_rcu(&con->node); + + /* + * Ensure that all SRCU list walks have completed so that the console + * can be added to the beginning of the console list and its forward + * list pointer can be re-initialized. + */ + synchronize_srcu(&console_srcu); + + con->flags |= CON_CONSDEV; + WARN_ON(!con->device); + + /* Only the new head can have CON_CONSDEV set. */ + console_srcu_write_flags(cur_pref_con, cur_pref_con->flags & ~CON_CONSDEV); + hlist_add_head_rcu(&con->node, &console_list); +} +EXPORT_SYMBOL(console_force_preferred_locked); + /* * Initialize the console device. This is called *early*, so * we can't necessarily depend on lots of kernel help here. -- cgit v1.2.3