From b6e43c0e3129ffe87e65c85f20fcbdf0eb86fba0 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 25 Oct 2019 17:42:10 +0100 Subject: arm64: remove __exception annotations Since commit 732674980139 ("arm64: unwind: reference pt_regs via embedded stack frame") arm64 has not used the __exception annotation to dump the pt_regs during stack tracing. in_exception_text() has no callers. This annotation is only used to blacklist kprobes, it means the same as __kprobes. Section annotations like this require the functions to be grouped together between the start/end markers, and placed according to the linker script. For kprobes we also have NOKPROBE_SYMBOL() which logs the symbol address in a section that kprobes parses and blacklists at boot. Using NOKPROBE_SYMBOL() instead lets kprobes publish the list of blacklisted symbols, and saves us from having an arm64 specific spelling of __kprobes. do_debug_exception() already has a NOKPROBE_SYMBOL() annotation. Signed-off-by: James Morse Acked-by: Mark Rutland Acked-by: Masami Hiramatsu Signed-off-by: Catalin Marinas --- arch/arm64/mm/fault.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) (limited to 'arch/arm64/mm') diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 855f2a7954e6..844cd2535826 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -732,8 +733,8 @@ static const struct fault_info fault_info[] = { { do_bad, SIGKILL, SI_KERNEL, "unknown 63" }, }; -asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, - struct pt_regs *regs) +asmlinkage void do_mem_abort(unsigned long addr, unsigned int esr, + struct pt_regs *regs) { const struct fault_info *inf = esr_to_fault_info(esr); @@ -749,16 +750,17 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, arm64_notify_die(inf->name, regs, inf->sig, inf->code, (void __user *)addr, esr); } +NOKPROBE_SYMBOL(do_mem_abort); -asmlinkage void __exception do_el0_irq_bp_hardening(void) +asmlinkage void do_el0_irq_bp_hardening(void) { /* PC has already been checked in entry.S */ arm64_apply_bp_hardening(); } +NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); -asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, - unsigned int esr, - struct pt_regs *regs) +asmlinkage void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, + struct pt_regs *regs) { /* * We've taken an instruction abort from userspace and not yet @@ -771,11 +773,10 @@ asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr, local_daif_restore(DAIF_PROCCTX); do_mem_abort(addr, esr, regs); } +NOKPROBE_SYMBOL(do_el0_ia_bp_hardening); - -asmlinkage void __exception do_sp_pc_abort(unsigned long addr, - unsigned int esr, - struct pt_regs *regs) +asmlinkage void do_sp_pc_abort(unsigned long addr, unsigned int esr, + struct pt_regs *regs) { if (user_mode(regs)) { if (!is_ttbr0_addr(instruction_pointer(regs))) @@ -786,6 +787,7 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN, (void __user *)addr, esr); } +NOKPROBE_SYMBOL(do_sp_pc_abort); int __init early_brk64(unsigned long addr, unsigned int esr, struct pt_regs *regs); @@ -868,8 +870,7 @@ NOKPROBE_SYMBOL(debug_exception_exit); #ifdef CONFIG_ARM64_ERRATUM_1463225 DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); -static int __exception -cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) +static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) { if (user_mode(regs)) return 0; @@ -888,16 +889,15 @@ cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) return 1; } #else -static int __exception -cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) +static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) { return 0; } #endif /* CONFIG_ARM64_ERRATUM_1463225 */ +NOKPROBE_SYMBOL(cortex_a76_erratum_1463225_debug_handler); -asmlinkage void __exception do_debug_exception(unsigned long addr_if_watchpoint, - unsigned int esr, - struct pt_regs *regs) +asmlinkage void do_debug_exception(unsigned long addr_if_watchpoint, + unsigned int esr, struct pt_regs *regs) { const struct fault_info *inf = esr_to_debug_fault_info(esr); unsigned long pc = instruction_pointer(regs); -- cgit v1.2.3 From afa7c0e5b965cdb945ad8a2e2973c6d7e19969f9 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 25 Oct 2019 17:42:15 +0100 Subject: arm64: Remove asmlinkage from updated functions Now that the callers of these functions have moved into C, they no longer need the asmlinkage annotation. Remove it. Signed-off-by: James Morse Acked-by: Mark Rutland Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/exception.h | 36 ++++++++++++++++-------------------- arch/arm64/kernel/fpsimd.c | 6 +++--- arch/arm64/kernel/syscall.c | 4 ++-- arch/arm64/kernel/traps.c | 8 ++++---- arch/arm64/mm/fault.c | 16 +++++++--------- 5 files changed, 32 insertions(+), 38 deletions(-) (limited to 'arch/arm64/mm') diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index a9e376623ecf..4d5f3b5f50cd 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -32,26 +32,22 @@ static inline u32 disr_to_esr(u64 disr) } asmlinkage void enter_from_user_mode(void); -asmlinkage void do_mem_abort(unsigned long addr, unsigned int esr, - struct pt_regs *regs); -asmlinkage void do_sp_pc_abort(unsigned long addr, unsigned int esr, - struct pt_regs *regs); -asmlinkage void do_undefinstr(struct pt_regs *regs); +void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs); +void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs); +void do_undefinstr(struct pt_regs *regs); asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr); -asmlinkage void do_debug_exception(unsigned long addr_if_watchpoint, - unsigned int esr, struct pt_regs *regs); -asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs); -asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs); -asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs); -asmlinkage void do_sysinstr(unsigned int esr, struct pt_regs *regs); -asmlinkage void do_sp_pc_abort(unsigned long addr, unsigned int esr, - struct pt_regs *regs); -asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, - unsigned int esr); -asmlinkage void do_cp15instr(unsigned int esr, struct pt_regs *regs); -asmlinkage void el0_svc_handler(struct pt_regs *regs); -asmlinkage void el0_svc_compat_handler(struct pt_regs *regs); -asmlinkage void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, - struct pt_regs *regs); +void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, + struct pt_regs *regs); +void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs); +void do_sve_acc(unsigned int esr, struct pt_regs *regs); +void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs); +void do_sysinstr(unsigned int esr, struct pt_regs *regs); +void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs); +void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr); +void do_cp15instr(unsigned int esr, struct pt_regs *regs); +void el0_svc_handler(struct pt_regs *regs); +void el0_svc_compat_handler(struct pt_regs *regs); +void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, + struct pt_regs *regs); #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 37d3912cfe06..3eb338f14386 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -920,7 +920,7 @@ void fpsimd_release_task(struct task_struct *dead_task) * would have disabled the SVE access trap for userspace during * ret_to_user, making an SVE access trap impossible in that case. */ -asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs) +void do_sve_acc(unsigned int esr, struct pt_regs *regs) { /* Even if we chose not to use SVE, the hardware could still trap: */ if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) { @@ -947,7 +947,7 @@ asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs) /* * Trapped FP/ASIMD access. */ -asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) +void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) { /* TODO: implement lazy context saving/restoring */ WARN_ON(1); @@ -956,7 +956,7 @@ asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) /* * Raise a SIGFPE for the current process. */ -asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) +void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) { unsigned int si_code = FPE_FLTUNK; diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 871c739f060a..9a9d98a443fc 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -154,14 +154,14 @@ static inline void sve_user_discard(void) sve_user_disable(); } -asmlinkage void el0_svc_handler(struct pt_regs *regs) +void el0_svc_handler(struct pt_regs *regs) { sve_user_discard(); el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table); } #ifdef CONFIG_COMPAT -asmlinkage void el0_svc_compat_handler(struct pt_regs *regs) +void el0_svc_compat_handler(struct pt_regs *regs) { el0_svc_common(regs, regs->regs[7], __NR_compat_syscalls, compat_sys_call_table); diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index ba1a571a7774..54ebe24ef4b1 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -394,7 +394,7 @@ void arm64_notify_segfault(unsigned long addr) force_signal_inject(SIGSEGV, code, addr); } -asmlinkage void do_undefinstr(struct pt_regs *regs) +void do_undefinstr(struct pt_regs *regs) { /* check for AArch32 breakpoint instructions */ if (!aarch32_break_handler(regs)) @@ -669,7 +669,7 @@ static const struct sys64_hook cp15_64_hooks[] = { {}, }; -asmlinkage void do_cp15instr(unsigned int esr, struct pt_regs *regs) +void do_cp15instr(unsigned int esr, struct pt_regs *regs) { const struct sys64_hook *hook, *hook_base; @@ -710,7 +710,7 @@ asmlinkage void do_cp15instr(unsigned int esr, struct pt_regs *regs) NOKPROBE_SYMBOL(do_cp15instr); #endif -asmlinkage void do_sysinstr(unsigned int esr, struct pt_regs *regs) +void do_sysinstr(unsigned int esr, struct pt_regs *regs) { const struct sys64_hook *hook; @@ -797,7 +797,7 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) * bad_el0_sync handles unexpected, but potentially recoverable synchronous * exceptions taken from EL0. Unlike bad_mode, this returns. */ -asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) +void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) { void __user *pc = (void __user *)instruction_pointer(regs); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 844cd2535826..cb13f4daa878 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -733,8 +733,7 @@ static const struct fault_info fault_info[] = { { do_bad, SIGKILL, SI_KERNEL, "unknown 63" }, }; -asmlinkage void do_mem_abort(unsigned long addr, unsigned int esr, - struct pt_regs *regs) +void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) { const struct fault_info *inf = esr_to_fault_info(esr); @@ -752,15 +751,15 @@ asmlinkage void do_mem_abort(unsigned long addr, unsigned int esr, } NOKPROBE_SYMBOL(do_mem_abort); -asmlinkage void do_el0_irq_bp_hardening(void) +void do_el0_irq_bp_hardening(void) { /* PC has already been checked in entry.S */ arm64_apply_bp_hardening(); } NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); -asmlinkage void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, - struct pt_regs *regs) +void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, + struct pt_regs *regs) { /* * We've taken an instruction abort from userspace and not yet @@ -775,8 +774,7 @@ asmlinkage void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, } NOKPROBE_SYMBOL(do_el0_ia_bp_hardening); -asmlinkage void do_sp_pc_abort(unsigned long addr, unsigned int esr, - struct pt_regs *regs) +void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) { if (user_mode(regs)) { if (!is_ttbr0_addr(instruction_pointer(regs))) @@ -896,8 +894,8 @@ static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) #endif /* CONFIG_ARM64_ERRATUM_1463225 */ NOKPROBE_SYMBOL(cortex_a76_erratum_1463225_debug_handler); -asmlinkage void do_debug_exception(unsigned long addr_if_watchpoint, - unsigned int esr, struct pt_regs *regs) +void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, + struct pt_regs *regs) { const struct fault_info *inf = esr_to_debug_fault_info(esr); unsigned long pc = instruction_pointer(regs); -- cgit v1.2.3 From bfe298745afc9548ad9344a9a3f26c81fd1a76c4 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 25 Oct 2019 17:42:16 +0100 Subject: arm64: entry-common: don't touch daif before bp-hardening The previous patches mechanically transformed the assembly version of entry.S to entry-common.c for synchronous exceptions. The C version of local_daif_restore() doesn't quite do the same thing as the assembly versions if pseudo-NMI is in use. In particular, | local_daif_restore(DAIF_PROCCTX_NOIRQ) will still allow pNMI to be delivered. This is not the behaviour do_el0_ia_bp_hardening() and do_sp_pc_abort() want as it should not be possible for the PMU handler to run as an NMI until the bp-hardening sequence has run. The bp-hardening calls were placed where they are because this was the first C code to run after the relevant exceptions. As we've now moved that point earlier, move the checks and calls earlier too. This makes it clearer that this stuff runs before any kind of exception, and saves modifying PSTATE twice. Signed-off-by: James Morse Reviewed-by: Mark Rutland Cc: Julien Thierry Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/processor.h | 14 ++++++++++++++ arch/arm64/kernel/entry-common.c | 18 +++++++++++++++--- arch/arm64/mm/fault.c | 36 +----------------------------------- 3 files changed, 30 insertions(+), 38 deletions(-) (limited to 'arch/arm64/mm') diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 5623685c7d13..8899d26f73ff 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -26,10 +26,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -214,6 +216,18 @@ static inline void start_thread(struct pt_regs *regs, unsigned long pc, regs->sp = sp; } +static inline bool is_ttbr0_addr(unsigned long addr) +{ + /* entry assembly clears tags for TTBR0 addrs */ + return addr < TASK_SIZE; +} + +static inline bool is_ttbr1_addr(unsigned long addr) +{ + /* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */ + return arch_kasan_reset_tag(addr) >= PAGE_OFFSET; +} + #ifdef CONFIG_COMPAT static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 2c318e41d84b..5dce5e56995a 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -14,6 +14,7 @@ #include #include #include +#include #include static void notrace el1_abort(struct pt_regs *regs, unsigned long esr) @@ -112,9 +113,17 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr) { unsigned long far = read_sysreg(far_el1); + /* + * We've taken an instruction abort from userspace and not yet + * re-enabled IRQs. If the address is a kernel address, apply + * BP hardening prior to enabling IRQs and pre-emption. + */ + if (!is_ttbr0_addr(far)) + arm64_apply_bp_hardening(); + user_exit_irqoff(); - local_daif_restore(DAIF_PROCCTX_NOIRQ); - do_el0_ia_bp_hardening(far, esr, regs); + local_daif_restore(DAIF_PROCCTX); + do_mem_abort(far, esr, regs); } NOKPROBE_SYMBOL(el0_ia); @@ -154,8 +163,11 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr) { unsigned long far = read_sysreg(far_el1); + if (!is_ttbr0_addr(instruction_pointer(regs))) + arm64_apply_bp_hardening(); + user_exit_irqoff(); - local_daif_restore(DAIF_PROCCTX_NOIRQ); + local_daif_restore(DAIF_PROCCTX); do_sp_pc_abort(far, esr, regs); } NOKPROBE_SYMBOL(el0_pc); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index cb13f4daa878..1bb2e3737e51 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -32,8 +32,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -102,18 +102,6 @@ static void mem_abort_decode(unsigned int esr) data_abort_decode(esr); } -static inline bool is_ttbr0_addr(unsigned long addr) -{ - /* entry assembly clears tags for TTBR0 addrs */ - return addr < TASK_SIZE; -} - -static inline bool is_ttbr1_addr(unsigned long addr) -{ - /* TTBR1 addresses may have a tag if KASAN_SW_TAGS is in use */ - return arch_kasan_reset_tag(addr) >= PAGE_OFFSET; -} - static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm) { /* Either init_pg_dir or swapper_pg_dir */ @@ -758,30 +746,8 @@ void do_el0_irq_bp_hardening(void) } NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); -void do_el0_ia_bp_hardening(unsigned long addr, unsigned int esr, - struct pt_regs *regs) -{ - /* - * We've taken an instruction abort from userspace and not yet - * re-enabled IRQs. If the address is a kernel address, apply - * BP hardening prior to enabling IRQs and pre-emption. - */ - if (!is_ttbr0_addr(addr)) - arm64_apply_bp_hardening(); - - local_daif_restore(DAIF_PROCCTX); - do_mem_abort(addr, esr, regs); -} -NOKPROBE_SYMBOL(do_el0_ia_bp_hardening); - void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - if (user_mode(regs)) { - if (!is_ttbr0_addr(instruction_pointer(regs))) - arm64_apply_bp_hardening(); - local_daif_restore(DAIF_PROCCTX); - } - arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN, (void __user *)addr, esr); } -- cgit v1.2.3