From 56afcd3dbd1995c526bfbd920cebde6158b22c4a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 23 Jun 2020 20:38:41 +0100 Subject: ARM: Allow IPIs to be handled as normal interrupts In order to deal with IPIs as normal interrupts, let's add a new way to register them with the architecture code. set_smp_ipi_range() takes a range of interrupts, and allows the arch code to request them as if the were normal interrupts. A standard handler is then called by the core IRQ code to deal with the IPI. This means that we don't need to call irq_enter/irq_exit, and that we don't need to deal with set_irq_regs either. So let's move the dispatcher into its own function, and leave handle_IPI() as a compatibility function. On the sending side, let's make use of ipi_send_mask, which already exists for this purpose. One of the major difference is that we end up, in some cases (such as when performing IRQ time accounting on the scheduler IPI), end up with nested irq_enter()/irq_exit() pairs. Other than the (relatively small) overhead, there should be no consequences to it (these pairs are designed to nest correctly, and the accounting shouldn't be off). Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- arch/arm/Kconfig | 1 + arch/arm/include/asm/smp.h | 5 +++ arch/arm/kernel/smp.c | 99 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 89 insertions(+), 16 deletions(-) (limited to 'arch/arm') diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e00d94b16658..e67ef15c800f 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -49,6 +49,7 @@ config ARM select GENERIC_ARCH_TOPOLOGY if ARM_CPU_TOPOLOGY select GENERIC_ATOMIC64 if CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI select GENERIC_CLOCKEVENTS_BROADCAST if SMP + select GENERIC_IRQ_IPI if SMP select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IDLE_POLL_SETUP diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index a91f21e3c5b5..0e29730295ca 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -45,6 +45,11 @@ extern void smp_init_cpus(void); */ extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); +/* + * Register IPI interrupts with the arch SMP code + */ +extern void set_smp_ipi_range(int ipi_base, int nr_ipi); + /* * Called from platform specific assembly code, this is the * secondary CPU entry point. diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 5d9da61eff62..f21f78483353 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -69,14 +69,22 @@ enum ipi_msg_type { * CPU_BACKTRACE is special and not included in NR_IPI * or tracable with trace_ipi_* */ - IPI_CPU_BACKTRACE, + IPI_CPU_BACKTRACE = NR_IPI, /* * SGI8-15 can be reserved by secure firmware, and thus may * not be usable by the kernel. Please keep the above limited * to at most 8 entries. */ + MAX_IPI }; +static int ipi_irq_base __read_mostly; +static int nr_ipi __read_mostly = NR_IPI; +static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly; + +static void ipi_setup(int cpu); +static void ipi_teardown(int cpu); + static DECLARE_COMPLETION(cpu_running); static struct smp_operations smp_ops __ro_after_init; @@ -247,6 +255,7 @@ int __cpu_disable(void) * and we must not schedule until we're ready to give up the cpu. */ set_cpu_online(cpu, false); + ipi_teardown(cpu); /* * OK - migrate IRQs away from this CPU @@ -422,6 +431,8 @@ asmlinkage void secondary_start_kernel(void) notify_cpu_starting(cpu); + ipi_setup(cpu); + calibrate_delay(); smp_store_cpu_info(cpu); @@ -627,10 +638,9 @@ asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) handle_IPI(ipinr, regs); } -void handle_IPI(int ipinr, struct pt_regs *regs) +static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - struct pt_regs *old_regs = set_irq_regs(regs); if ((unsigned)ipinr < NR_IPI) { trace_ipi_entry_rcuidle(ipi_types[ipinr]); @@ -643,9 +653,7 @@ void handle_IPI(int ipinr, struct pt_regs *regs) #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST case IPI_TIMER: - irq_enter(); tick_receive_broadcast(); - irq_exit(); break; #endif @@ -654,36 +662,26 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; case IPI_CALL_FUNC: - irq_enter(); generic_smp_call_function_interrupt(); - irq_exit(); break; case IPI_CPU_STOP: - irq_enter(); ipi_cpu_stop(cpu); - irq_exit(); break; #ifdef CONFIG_IRQ_WORK case IPI_IRQ_WORK: - irq_enter(); irq_work_run(); - irq_exit(); break; #endif case IPI_COMPLETION: - irq_enter(); ipi_complete(cpu); - irq_exit(); break; case IPI_CPU_BACKTRACE: printk_nmi_enter(); - irq_enter(); - nmi_cpu_backtrace(regs); - irq_exit(); + nmi_cpu_backtrace(get_irq_regs()); printk_nmi_exit(); break; @@ -695,9 +693,78 @@ void handle_IPI(int ipinr, struct pt_regs *regs) if ((unsigned)ipinr < NR_IPI) trace_ipi_exit_rcuidle(ipi_types[ipinr]); +} + +/* Legacy version, should go away once all irqchips have been converted */ +void handle_IPI(int ipinr, struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + + irq_enter(); + do_handle_IPI(ipinr); + irq_exit(); + set_irq_regs(old_regs); } +static irqreturn_t ipi_handler(int irq, void *data) +{ + do_handle_IPI(irq - ipi_irq_base); + return IRQ_HANDLED; +} + +static void ipi_send(const struct cpumask *target, unsigned int ipi) +{ + __ipi_send_mask(ipi_desc[ipi], target); +} + +static void ipi_setup(int cpu) +{ + int i; + + if (!ipi_irq_base) + return; + + for (i = 0; i < nr_ipi; i++) + enable_percpu_irq(ipi_irq_base + i, 0); +} + +static void ipi_teardown(int cpu) +{ + int i; + + if (!ipi_irq_base) + return; + + for (i = 0; i < nr_ipi; i++) + disable_percpu_irq(ipi_irq_base + i); +} + +void __init set_smp_ipi_range(int ipi_base, int n) +{ + int i; + + WARN_ON(n < MAX_IPI); + nr_ipi = min(n, MAX_IPI); + + for (i = 0; i < nr_ipi; i++) { + int err; + + err = request_percpu_irq(ipi_base + i, ipi_handler, + "IPI", &irq_stat); + WARN_ON(err); + + ipi_desc[i] = irq_to_desc(ipi_base + i); + irq_set_status_flags(ipi_base + i, IRQ_HIDDEN); + } + + ipi_irq_base = ipi_base; + set_smp_cross_call(ipi_send); + + /* Setup the boot CPU immediately */ + ipi_setup(smp_processor_id()); +} + void smp_send_reschedule(int cpu) { smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); -- cgit v1.2.3 From 8aa837cb7a032884c787b15de81f7d9de8af0869 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 22 Jun 2020 22:15:54 +0100 Subject: ARM: Kill __smp_cross_call and co The old IPI registration interface is now unused on arm, so let's get rid of it. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- arch/arm/include/asm/smp.h | 6 ------ arch/arm/kernel/smp.c | 26 +++++++------------------- 2 files changed, 7 insertions(+), 25 deletions(-) (limited to 'arch/arm') diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 0e29730295ca..0ca55a607d0a 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -39,12 +39,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs); */ extern void smp_init_cpus(void); - -/* - * Provide a function to raise an IPI cross call on CPUs in callmap. - */ -extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int)); - /* * Register IPI interrupts with the arch SMP code */ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index f21f78483353..d51e64955a26 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -511,14 +511,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) } } -static void (*__smp_cross_call)(const struct cpumask *, unsigned int); - -void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) -{ - if (!__smp_cross_call) - __smp_cross_call = fn; -} - static const char *ipi_types[NR_IPI] __tracepoint_string = { #define S(x,s) [x] = s S(IPI_WAKEUP, "CPU wakeup interrupts"), @@ -530,11 +522,7 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { S(IPI_COMPLETION, "completion interrupts"), }; -static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) -{ - trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); - __smp_cross_call(target, ipinr); -} +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr); void show_ipi_list(struct seq_file *p, int prec) { @@ -713,16 +701,17 @@ static irqreturn_t ipi_handler(int irq, void *data) return IRQ_HANDLED; } -static void ipi_send(const struct cpumask *target, unsigned int ipi) +static void smp_cross_call(const struct cpumask *target, unsigned int ipinr) { - __ipi_send_mask(ipi_desc[ipi], target); + trace_ipi_raise_rcuidle(target, ipi_types[ipinr]); + __ipi_send_mask(ipi_desc[ipinr], target); } static void ipi_setup(int cpu) { int i; - if (!ipi_irq_base) + if (WARN_ON_ONCE(!ipi_irq_base)) return; for (i = 0; i < nr_ipi; i++) @@ -733,7 +722,7 @@ static void ipi_teardown(int cpu) { int i; - if (!ipi_irq_base) + if (WARN_ON_ONCE(!ipi_irq_base)) return; for (i = 0; i < nr_ipi; i++) @@ -759,7 +748,6 @@ void __init set_smp_ipi_range(int ipi_base, int n) } ipi_irq_base = ipi_base; - set_smp_cross_call(ipi_send); /* Setup the boot CPU immediately */ ipi_setup(smp_processor_id()); @@ -872,7 +860,7 @@ core_initcall(register_cpufreq_notifier); static void raise_nmi(cpumask_t *mask) { - __smp_cross_call(mask, IPI_CPU_BACKTRACE); + __ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask); } void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) -- cgit v1.2.3 From 5ebf353af22c89d18964bb3b877a95200dfe07b9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 23 Jun 2020 21:15:00 +0100 Subject: ARM: Remove custom IRQ stat accounting Let's switch the arm code to the core accounting, which already does everything we need. Reviewed-by: Valentin Schneider Signed-off-by: Marc Zyngier --- arch/arm/include/asm/hardirq.h | 17 ----------------- arch/arm/kernel/irq.c | 1 - arch/arm/kernel/smp.c | 21 +++++---------------- 3 files changed, 5 insertions(+), 34 deletions(-) (limited to 'arch/arm') diff --git a/arch/arm/include/asm/hardirq.h b/arch/arm/include/asm/hardirq.h index 7a88f160b1fb..b95848ed2bc7 100644 --- a/arch/arm/include/asm/hardirq.h +++ b/arch/arm/include/asm/hardirq.h @@ -6,29 +6,12 @@ #include #include -/* number of IPIS _not_ including IPI_CPU_BACKTRACE */ -#define NR_IPI 7 - typedef struct { unsigned int __softirq_pending; -#ifdef CONFIG_SMP - unsigned int ipi_irqs[NR_IPI]; -#endif } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ -#define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ -#define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) - -#ifdef CONFIG_SMP -u64 smp_irq_stat_cpu(unsigned int cpu); -#else -#define smp_irq_stat_cpu(cpu) 0 -#endif - -#define arch_irq_stat_cpu smp_irq_stat_cpu - #define __ARCH_IRQ_EXIT_IRQS_DISABLED 1 #endif /* __ASM_HARDIRQ_H */ diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index ee514034c0a1..698b6f636156 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -18,7 +18,6 @@ * IRQ's are in fact implemented a bit like signal handlers for the kernel. * Naturally it's not a 1:1 relation, but there are similarities. */ -#include #include #include #include diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index d51e64955a26..00327fa74b01 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -65,6 +66,7 @@ enum ipi_msg_type { IPI_CPU_STOP, IPI_IRQ_WORK, IPI_COMPLETION, + NR_IPI, /* * CPU_BACKTRACE is special and not included in NR_IPI * or tracable with trace_ipi_* @@ -529,27 +531,16 @@ void show_ipi_list(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { + unsigned int irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) - seq_printf(p, "%10u ", - __get_irq_stat(cpu, ipi_irqs[i])); + seq_printf(p, "%10u ", kstat_irqs_cpu(irq, cpu)); seq_printf(p, " %s\n", ipi_types[i]); } } -u64 smp_irq_stat_cpu(unsigned int cpu) -{ - u64 sum = 0; - int i; - - for (i = 0; i < NR_IPI; i++) - sum += __get_irq_stat(cpu, ipi_irqs[i]); - - return sum; -} - void arch_send_call_function_ipi_mask(const struct cpumask *mask) { smp_cross_call(mask, IPI_CALL_FUNC); @@ -630,10 +621,8 @@ static void do_handle_IPI(int ipinr) { unsigned int cpu = smp_processor_id(); - if ((unsigned)ipinr < NR_IPI) { + if ((unsigned)ipinr < NR_IPI) trace_ipi_entry_rcuidle(ipi_types[ipinr]); - __inc_irq_stat(cpu, ipi_irqs[ipinr]); - } switch (ipinr) { case IPI_WAKEUP: -- cgit v1.2.3 From ac15a54e03d13686d2fc016a88311801b0734046 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 18 Sep 2020 17:19:46 +0100 Subject: arm: Move ipi_teardown() to a CONFIG_HOTPLUG_CPU section ipi_teardown() is only used when CONFIG_HOTPLUG_CPU is enabled. Move the function to a location guarded by this config option. Signed-off-by: Marc Zyngier --- arch/arm/kernel/smp.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'arch/arm') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 00327fa74b01..8425da517984 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -85,7 +85,6 @@ static int nr_ipi __read_mostly = NR_IPI; static struct irq_desc *ipi_desc[MAX_IPI] __read_mostly; static void ipi_setup(int cpu); -static void ipi_teardown(int cpu); static DECLARE_COMPLETION(cpu_running); @@ -236,6 +235,17 @@ int platform_can_hotplug_cpu(unsigned int cpu) return cpu != 0; } +static void ipi_teardown(int cpu) +{ + int i; + + if (WARN_ON_ONCE(!ipi_irq_base)) + return; + + for (i = 0; i < nr_ipi; i++) + disable_percpu_irq(ipi_irq_base + i); +} + /* * __cpu_disable runs on the processor to be shutdown. */ @@ -707,17 +717,6 @@ static void ipi_setup(int cpu) enable_percpu_irq(ipi_irq_base + i, 0); } -static void ipi_teardown(int cpu) -{ - int i; - - if (WARN_ON_ONCE(!ipi_irq_base)) - return; - - for (i = 0; i < nr_ipi; i++) - disable_percpu_irq(ipi_irq_base + i); -} - void __init set_smp_ipi_range(int ipi_base, int n) { int i; -- cgit v1.2.3 From 220387048d859896ccc362c0ebf9bc1e0fa62eb9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 25 Sep 2020 16:22:00 +0100 Subject: ARM: Handle no IPI being registered in show_ipi_list() As SMP-on-UP is a valid configuration on 32bit ARM, do not assume that IPIs are populated in show_ipi_list(). Reported-by: Guillaume Tucker Reported-by: kernelci.org bot Tested-by: Guillaume Tucker Signed-off-by: Marc Zyngier --- arch/arm/kernel/smp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'arch/arm') diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 8425da517984..48099c6e1e4a 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -541,7 +541,12 @@ void show_ipi_list(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { - unsigned int irq = irq_desc_get_irq(ipi_desc[i]); + unsigned int irq; + + if (!ipi_desc[i]) + continue; + + irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) -- cgit v1.2.3