From f25d384755191690b1196776d319cb6a4e899f28 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Thu, 9 Feb 2017 01:34:49 +0100 Subject: x86/asm: Optimize clear_page() Currently, we CALL clear_page() which then JMPs to the proper function chosen by the alternatives. What we should do instead is CALL the proper function directly. (This was something Ingo suggested a while ago). So let's do that. Measuring our favourite kernel build workload shows that there are no significant changes in performance. AMD === -- /tmp/before 2017-02-09 18:01:46.451961188 +0100 ++ /tmp/after 2017-02-09 18:01:54.883961175 +0100 @@ -1,15 +1,15 @@ Performance counter stats for 'system wide' (5 runs): - 1028960.373643 cpu-clock (msec) # 6.000 CPUs utilized ( +- 1.41% ) + 1023086.018961 cpu-clock (msec) # 6.000 CPUs utilized ( +- 1.20% ) - 518,744 context-switches # 0.504 K/sec ( +- 1.04% ) + 518,254 context-switches # 0.507 K/sec ( +- 1.01% ) - 38,112 cpu-migrations # 0.037 K/sec ( +- 1.95% ) + 37,917 cpu-migrations # 0.037 K/sec ( +- 1.02% ) - 20,874,266 page-faults # 0.020 M/sec ( +- 0.07% ) + 20,918,897 page-faults # 0.020 M/sec ( +- 0.18% ) - 2,043,646,230,667 cycles # 1.986 GHz ( +- 0.14% ) (66.67%) + 2,045,305,584,032 cycles # 1.999 GHz ( +- 0.16% ) (66.67%) - 553,698,855,431 stalled-cycles-frontend # 27.09% frontend cycles idle ( +- 0.07% ) (66.67%) + 555,099,401,413 stalled-cycles-frontend # 27.14% frontend cycles idle ( +- 0.13% ) (66.67%) - 621,544,286,390 stalled-cycles-backend # 30.41% backend cycles idle ( +- 0.39% ) (66.67%) + 621,371,430,254 stalled-cycles-backend # 30.38% backend cycles idle ( +- 0.32% ) (66.67%) - 1,738,364,431,659 instructions # 0.85 insn per cycle + 1,739,895,771,901 instructions # 0.85 insn per cycle - # 0.36 stalled cycles per insn ( +- 0.11% ) (66.67%) + # 0.36 stalled cycles per insn ( +- 0.13% ) (66.67%) - 391,170,943,850 branches # 380.161 M/sec ( +- 0.13% ) (66.67%) + 391,398,551,757 branches # 382.567 M/sec ( +- 0.13% ) (66.67%) - 22,567,810,411 branch-misses # 5.77% of all branches ( +- 0.11% ) (66.67%) + 22,574,726,683 branch-misses # 5.77% of all branches ( +- 0.13% ) (66.67%) - 171.480741921 seconds time elapsed ( +- 1.41% ) + 170.509229451 seconds time elapsed ( +- 1.20% ) Intel ===== -- /tmp/before 2017-02-09 20:36:19.851947473 +0100 ++ /tmp/after 2017-02-09 20:36:30.151947458 +0100 @@ -1,15 +1,15 @@ Performance counter stats for 'system wide' (5 runs): - 2207248.598126 cpu-clock (msec) # 8.000 CPUs utilized ( +- 0.69% ) + 2213300.106631 cpu-clock (msec) # 8.000 CPUs utilized ( +- 0.73% ) - 899,342 context-switches # 0.407 K/sec ( +- 0.68% ) + 898,381 context-switches # 0.406 K/sec ( +- 0.79% ) - 80,553 cpu-migrations # 0.036 K/sec ( +- 1.13% ) + 80,979 cpu-migrations # 0.037 K/sec ( +- 1.11% ) - 36,171,148 page-faults # 0.016 M/sec ( +- 0.02% ) + 36,179,791 page-faults # 0.016 M/sec ( +- 0.02% ) - 6,665,288,826,484 cycles # 3.020 GHz ( +- 0.07% ) (83.33%) + 6,671,638,410,799 cycles # 3.014 GHz ( +- 0.06% ) (83.33%) - 5,065,975,115,197 stalled-cycles-frontend # 76.01% frontend cycles idle ( +- 0.11% ) (83.33%) + 5,076,835,183,223 stalled-cycles-frontend # 76.10% frontend cycles idle ( +- 0.11% ) (83.33%) - 3,841,556,350,614 stalled-cycles-backend # 57.64% backend cycles idle ( +- 0.13% ) (66.67%) + 3,852,823,974,333 stalled-cycles-backend # 57.75% backend cycles idle ( +- 0.12% ) (66.67%) - 4,148,398,171,079 instructions # 0.62 insn per cycle + 4,148,997,156,059 instructions # 0.62 insn per cycle - # 1.22 stalled cycles per insn ( +- 0.10% ) (83.33%) + # 1.22 stalled cycles per insn ( +- 0.11% ) (83.33%) - 887,187,118,591 branches # 401.943 M/sec ( +- 0.09% ) (83.33%) + 887,271,341,121 branches # 400.882 M/sec ( +- 0.11% ) (83.33%) - 30,139,439,034 branch-misses # 3.40% of all branches ( +- 0.09% ) (83.33%) + 30,134,864,997 branch-misses # 3.40% of all branches ( +- 0.06% ) (83.33%) - 275.904405540 seconds time elapsed ( +- 0.69% ) + 276.660352016 seconds time elapsed ( +- 0.73% ) allmodconfig vmlinux size grows by a ~1Kb but that's fine - we optimize our calling of the clear_page variants. text data bss dec hex filename 9051979 23067670 27009024 59128673 3863b61 vmlinux 9053000 23067670 27009024 59129694 3863f5e vmlinux.clear_page Reported-by: kernel test robot Tested-by: Fengguang Wu Signed-off-by: Borislav Petkov Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20170215111927.emdgxf2pide3kwro@pd.tnic Signed-off-by: Ingo Molnar --- arch/x86/include/asm/page_64.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'arch/x86/include/asm') diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h index b3bebf9e5746..b4a0d43248cf 100644 --- a/arch/x86/include/asm/page_64.h +++ b/arch/x86/include/asm/page_64.h @@ -4,6 +4,7 @@ #include #ifndef __ASSEMBLY__ +#include /* duplicated to the one in bootmem.h */ extern unsigned long max_pfn; @@ -34,7 +35,20 @@ extern unsigned long __phys_addr_symbol(unsigned long); #define pfn_valid(pfn) ((pfn) < max_pfn) #endif -void clear_page(void *page); +void clear_page_orig(void *page); +void clear_page_rep(void *page); +void clear_page_erms(void *page); + +static inline void clear_page(void *page) +{ + alternative_call_2(clear_page_orig, + clear_page_rep, X86_FEATURE_REP_GOOD, + clear_page_erms, X86_FEATURE_ERMS, + "=D" (page), + "0" (page) + : "memory", "rax", "rcx"); +} + void copy_page(void *to, void *from); #endif /* !__ASSEMBLY__ */ -- cgit v1.2.3 From 9a93848fe787a53aec404e4e00d8f7f9bbdaebb4 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 2 Feb 2017 14:43:51 +0100 Subject: x86/debug: Implement __WARN() using UD0 By using "UD0" for WARN()s we remove the function call and its possible __FILE__ and __LINE__ immediate arguments from the instruction stream. Total image size will not change much, what we win in the instruction stream we'll lose because of the __bug_table entries. Still, saves on I$ footprint and the total image size does go down a bit. text data filename 10702123 4530992 defconfig-build/vmlinux.orig 10682460 4530992 defconfig-build/vmlinux.patched (UML didn't seem to use GENERIC_BUG at all, so remove it) Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Josh Poimboeuf Cc: Arjan van de Ven Cc: Borislav Petkov Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Richard Weinberger Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- arch/um/Kconfig.common | 5 --- arch/x86/include/asm/bug.h | 78 ++++++++++++++++++++++++++++++++---------- arch/x86/kernel/dumpstack.c | 3 -- arch/x86/kernel/dumpstack_32.c | 12 ------- arch/x86/kernel/dumpstack_64.c | 10 ------ arch/x86/kernel/traps.c | 46 +++++++++++++++++++++---- arch/x86/um/Makefile | 2 +- arch/x86/um/bug.c | 21 ------------ 8 files changed, 101 insertions(+), 76 deletions(-) delete mode 100644 arch/x86/um/bug.c (limited to 'arch/x86/include/asm') diff --git a/arch/um/Kconfig.common b/arch/um/Kconfig.common index fd443852103c..ed9c5b5ff028 100644 --- a/arch/um/Kconfig.common +++ b/arch/um/Kconfig.common @@ -50,11 +50,6 @@ config GENERIC_CALIBRATE_DELAY bool default y -config GENERIC_BUG - bool - default y - depends on BUG - config HZ int default 100 diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index ba38ebbaced3..4fde330c44b7 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -1,36 +1,78 @@ #ifndef _ASM_X86_BUG_H #define _ASM_X86_BUG_H -#define HAVE_ARCH_BUG +#include -#ifdef CONFIG_DEBUG_BUGVERBOSE +/* + * Since some emulators terminate on UD2, we cannot use it for WARN. + * Since various instruction decoders disagree on the length of UD1, + * we cannot use it either. So use UD0 for WARN. + * + * (binutils knows about "ud1" but {en,de}codes it as 2 bytes, whereas + * our kernel decoder thinks it takes a ModRM byte, which seems consistent + * with various things like the Intel SDM instruction encoding rules) + */ + +#define ASM_UD0 ".byte 0x0f, 0xff" +#define ASM_UD1 ".byte 0x0f, 0xb9" /* + ModRM */ +#define ASM_UD2 ".byte 0x0f, 0x0b" + +#define INSN_UD0 0xff0f +#define INSN_UD2 0x0b0f + +#define LEN_UD0 2 + +#ifdef CONFIG_GENERIC_BUG +#define HAVE_ARCH_BUG #ifdef CONFIG_X86_32 -# define __BUG_C0 "2:\t.long 1b, %c0\n" +# define __BUG_REL(val) ".long " __stringify(val) #else -# define __BUG_C0 "2:\t.long 1b - 2b, %c0 - 2b\n" +# define __BUG_REL(val) ".long " __stringify(val) " - 2b" #endif -#define BUG() \ -do { \ - asm volatile("1:\tud2\n" \ - ".pushsection __bug_table,\"a\"\n" \ - __BUG_C0 \ - "\t.word %c1, 0\n" \ - "\t.org 2b+%c2\n" \ - ".popsection" \ - : : "i" (__FILE__), "i" (__LINE__), \ - "i" (sizeof(struct bug_entry))); \ - unreachable(); \ +#ifdef CONFIG_DEBUG_BUGVERBOSE + +#define _BUG_FLAGS(ins, flags) \ +do { \ + asm volatile("1:\t" ins "\n" \ + ".pushsection __bug_table,\"a\"\n" \ + "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n" \ + "\t" __BUG_REL(%c0) "\t# bug_entry::file\n" \ + "\t.word %c1" "\t# bug_entry::line\n" \ + "\t.word %c2" "\t# bug_entry::flags\n" \ + "\t.org 2b+%c3\n" \ + ".popsection" \ + : : "i" (__FILE__), "i" (__LINE__), \ + "i" (flags), \ + "i" (sizeof(struct bug_entry))); \ } while (0) -#else +#else /* !CONFIG_DEBUG_BUGVERBOSE */ + +#define _BUG_FLAGS(ins, flags) \ +do { \ + asm volatile("1:\t" ins "\n" \ + ".pushsection __bug_table,\"a\"\n" \ + "2:\t" __BUG_REL(1b) "\t# bug_entry::bug_addr\n" \ + "\t.word %c0" "\t# bug_entry::flags\n" \ + "\t.org 2b+%c1\n" \ + ".popsection" \ + : : "i" (flags), \ + "i" (sizeof(struct bug_entry))); \ +} while (0) + +#endif /* CONFIG_DEBUG_BUGVERBOSE */ + #define BUG() \ do { \ - asm volatile("ud2"); \ + _BUG_FLAGS(ASM_UD2, 0); \ unreachable(); \ } while (0) -#endif + +#define __WARN_TAINT(taint) _BUG_FLAGS(ASM_UD0, BUGFLAG_TAINT(taint)) + +#endif /* CONFIG_GENERIC_BUG */ #include diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 09d4ac0d2661..924f45ea4382 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -289,9 +289,6 @@ void die(const char *str, struct pt_regs *regs, long err) unsigned long flags = oops_begin(); int sig = SIGSEGV; - if (!user_mode(regs)) - report_bug(regs->ip, regs); - if (__die(str, regs, err)) sig = 0; oops_end(flags, regs, sig); diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c index b0b3a3df7c20..e5f0b40e66d2 100644 --- a/arch/x86/kernel/dumpstack_32.c +++ b/arch/x86/kernel/dumpstack_32.c @@ -162,15 +162,3 @@ void show_regs(struct pt_regs *regs) } pr_cont("\n"); } - -int is_valid_bugaddr(unsigned long ip) -{ - unsigned short ud2; - - if (ip < PAGE_OFFSET) - return 0; - if (probe_kernel_address((unsigned short *)ip, ud2)) - return 0; - - return ud2 == 0x0b0f; -} diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index a8b117e93b46..3e1471d57487 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -178,13 +178,3 @@ void show_regs(struct pt_regs *regs) } pr_cont("\n"); } - -int is_valid_bugaddr(unsigned long ip) -{ - unsigned short ud2; - - if (__copy_from_user(&ud2, (const void __user *) ip, sizeof(ud2))) - return 0; - - return ud2 == 0x0b0f; -} diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 948443e115c1..3c0751b120de 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -169,6 +169,37 @@ void ist_end_non_atomic(void) preempt_disable(); } +int is_valid_bugaddr(unsigned long addr) +{ + unsigned short ud; + + if (addr < TASK_SIZE_MAX) + return 0; + + if (probe_kernel_address((unsigned short *)addr, ud)) + return 0; + + return ud == INSN_UD0 || ud == INSN_UD2; +} + +static int fixup_bug(struct pt_regs *regs, int trapnr) +{ + if (trapnr != X86_TRAP_UD) + return 0; + + switch (report_bug(regs->ip, regs)) { + case BUG_TRAP_TYPE_NONE: + case BUG_TRAP_TYPE_BUG: + break; + + case BUG_TRAP_TYPE_WARN: + regs->ip += LEN_UD0; + return 1; + } + + return 0; +} + static nokprobe_inline int do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str, struct pt_regs *regs, long error_code) @@ -187,12 +218,15 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str, } if (!user_mode(regs)) { - if (!fixup_exception(regs, trapnr)) { - tsk->thread.error_code = error_code; - tsk->thread.trap_nr = trapnr; - die(str, regs, error_code); - } - return 0; + if (fixup_exception(regs, trapnr)) + return 0; + + if (fixup_bug(regs, trapnr)) + return 0; + + tsk->thread.error_code = error_code; + tsk->thread.trap_nr = trapnr; + die(str, regs, error_code); } return -1; diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile index e7e7055a8658..76f17f05446f 100644 --- a/arch/x86/um/Makefile +++ b/arch/x86/um/Makefile @@ -8,7 +8,7 @@ else BITS := 64 endif -obj-y = bug.o bugs_$(BITS).o delay.o fault.o ldt.o \ +obj-y = bugs_$(BITS).o delay.o fault.o ldt.o \ ptrace_$(BITS).o ptrace_user.o setjmp_$(BITS).o signal.o \ stub_$(BITS).o stub_segv.o \ sys_call_table_$(BITS).o sysrq_$(BITS).o tls_$(BITS).o \ diff --git a/arch/x86/um/bug.c b/arch/x86/um/bug.c deleted file mode 100644 index e8034e363d83..000000000000 --- a/arch/x86/um/bug.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2006 Jeff Dike (jdike@addtoit.com) - * Licensed under the GPL V2 - */ - -#include - -/* - * Mostly copied from i386/x86_86 - eliminated the eip < PAGE_OFFSET because - * that's not relevant in skas mode. - */ - -int is_valid_bugaddr(unsigned long eip) -{ - unsigned short ud2; - - if (probe_kernel_address((unsigned short __user *)eip, ud2)) - return 0; - - return ud2 == 0x0b0f; -} -- cgit v1.2.3 From 70579a86e3c8eb2ce57999e594a73b4dfe095959 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 29 Mar 2017 23:16:31 +0200 Subject: x86/debug: Define BUG() again for !CONFIG_BUG The latest change to the BUG() macro inadvertently reverted the earlier commit: b06dd879f5db ("x86: always define BUG() and HAVE_ARCH_BUG, even with !CONFIG_BUG") ... that sanitized the behavior with CONFIG_BUG=n. I noticed this as some warnings have appeared again that were previously fixed as a side effect of that patch: kernel/seccomp.c: In function '__seccomp_filter': kernel/seccomp.c:670:1: error: no return statement in function returning non-void [-Werror=return-type] ... This combines the two patches and uses the ud2 macro to define BUG() in case of CONFIG_BUG=n. Signed-off-by: Arnd Bergmann Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Josh Poimboeuf Cc: Josh Triplett Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Fixes: 9a93848fe787 ("x86/debug: Implement __WARN() using UD0") Link: http://lkml.kernel.org/r/20170329211646.2707365-1-arnd@arndb.de Signed-off-by: Ingo Molnar --- arch/x86/include/asm/bug.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'arch/x86/include/asm') diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index 4fde330c44b7..cecf559d0012 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -23,7 +23,6 @@ #define LEN_UD0 2 #ifdef CONFIG_GENERIC_BUG -#define HAVE_ARCH_BUG #ifdef CONFIG_X86_32 # define __BUG_REL(val) ".long " __stringify(val) @@ -64,6 +63,13 @@ do { \ #endif /* CONFIG_DEBUG_BUGVERBOSE */ +#else + +#define _BUG_FLAGS(ins, flags) asm volatile(ins) + +#endif /* CONFIG_GENERIC_BUG */ + +#define HAVE_ARCH_BUG #define BUG() \ do { \ _BUG_FLAGS(ASM_UD2, 0); \ @@ -72,8 +78,6 @@ do { \ #define __WARN_TAINT(taint) _BUG_FLAGS(ASM_UD0, BUGFLAG_TAINT(taint)) -#endif /* CONFIG_GENERIC_BUG */ - #include #endif /* _ASM_X86_BUG_H */ -- cgit v1.2.3 From 19d436268dde95389c616bb3819da73f0a8b28a8 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Sat, 25 Feb 2017 08:56:53 +0100 Subject: debug: Add _ONCE() logic to report_bug() Josh suggested moving the _ONCE logic inside the trap handler, using a bit in the bug_entry::flags field, avoiding the need for the extra variable. Sadly this only works for WARN_ON_ONCE(), since the others have printk() statements prior to triggering the trap. Still, this saves a fair amount of text and some data: text data filename 10682460 4530992 defconfig-build/vmlinux.orig 10665111 4530096 defconfig-build/vmlinux.patched Suggested-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Cc: Andy Lutomirski Cc: Arnd Bergmann Cc: Borislav Petkov Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/arm64/include/asm/bug.h | 2 +- arch/parisc/include/asm/bug.h | 8 ++++---- arch/powerpc/include/asm/bug.h | 4 ++-- arch/s390/include/asm/bug.h | 4 ++-- arch/sh/include/asm/bug.h | 4 ++-- arch/x86/include/asm/bug.h | 2 +- include/asm-generic/bug.h | 18 +++++++++++++++++- include/asm-generic/vmlinux.lds.h | 5 ++--- include/linux/bug.h | 2 +- lib/bug.c | 28 ++++++++++++++++++++-------- 10 files changed, 52 insertions(+), 25 deletions(-) (limited to 'arch/x86/include/asm') diff --git a/arch/arm64/include/asm/bug.h b/arch/arm64/include/asm/bug.h index 561190d15881..a9be1072933c 100644 --- a/arch/arm64/include/asm/bug.h +++ b/arch/arm64/include/asm/bug.h @@ -55,7 +55,7 @@ _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ unreachable(); \ } while (0) -#define __WARN_TAINT(taint) _BUG_FLAGS(BUGFLAG_TAINT(taint)) +#define __WARN_FLAGS(flags) _BUG_FLAGS(BUGFLAG_WARNING|(flags)) #endif /* ! CONFIG_GENERIC_BUG */ diff --git a/arch/parisc/include/asm/bug.h b/arch/parisc/include/asm/bug.h index 62a33338549c..d2742273a685 100644 --- a/arch/parisc/include/asm/bug.h +++ b/arch/parisc/include/asm/bug.h @@ -46,7 +46,7 @@ #endif #ifdef CONFIG_DEBUG_BUGVERBOSE -#define __WARN_TAINT(taint) \ +#define __WARN_FLAGS(flags) \ do { \ asm volatile("\n" \ "1:\t" PARISC_BUG_BREAK_ASM "\n" \ @@ -56,11 +56,11 @@ "\t.org 2b+%c3\n" \ "\t.popsection" \ : : "i" (__FILE__), "i" (__LINE__), \ - "i" (BUGFLAG_TAINT(taint)), \ + "i" (BUGFLAG_WARNING|(flags)), \ "i" (sizeof(struct bug_entry)) ); \ } while(0) #else -#define __WARN_TAINT(taint) \ +#define __WARN_FLAGS(flags) \ do { \ asm volatile("\n" \ "1:\t" PARISC_BUG_BREAK_ASM "\n" \ @@ -69,7 +69,7 @@ "\t.short %c0\n" \ "\t.org 2b+%c1\n" \ "\t.popsection" \ - : : "i" (BUGFLAG_TAINT(taint)), \ + : : "i" (BUGFLAG_WARNING|(flags)), \ "i" (sizeof(struct bug_entry)) ); \ } while(0) #endif diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h index 3a39283333c3..f2c562a0a427 100644 --- a/arch/powerpc/include/asm/bug.h +++ b/arch/powerpc/include/asm/bug.h @@ -85,12 +85,12 @@ } \ } while (0) -#define __WARN_TAINT(taint) do { \ +#define __WARN_FLAGS(flags) do { \ __asm__ __volatile__( \ "1: twi 31,0,0\n" \ _EMIT_BUG_ENTRY \ : : "i" (__FILE__), "i" (__LINE__), \ - "i" (BUGFLAG_TAINT(taint)), \ + "i" (BUGFLAG_WARNING|(flags)), \ "i" (sizeof(struct bug_entry))); \ } while (0) diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index bf90d1fd97a5..1bbd9dbfe4e0 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -46,8 +46,8 @@ unreachable(); \ } while (0) -#define __WARN_TAINT(taint) do { \ - __EMIT_BUG(BUGFLAG_TAINT(taint)); \ +#define __WARN_FLAGS(flags) do { \ + __EMIT_BUG(BUGFLAG_WARNING|(flags)); \ } while (0) #define WARN_ON(x) ({ \ diff --git a/arch/sh/include/asm/bug.h b/arch/sh/include/asm/bug.h index dcf278075429..1b77f068be2b 100644 --- a/arch/sh/include/asm/bug.h +++ b/arch/sh/include/asm/bug.h @@ -50,7 +50,7 @@ do { \ "i" (sizeof(struct bug_entry))); \ } while (0) -#define __WARN_TAINT(taint) \ +#define __WARN_FLAGS(flags) \ do { \ __asm__ __volatile__ ( \ "1:\t.short %O0\n" \ @@ -59,7 +59,7 @@ do { \ : "n" (TRAPA_BUG_OPCODE), \ "i" (__FILE__), \ "i" (__LINE__), \ - "i" (BUGFLAG_TAINT(taint)), \ + "i" (BUGFLAG_WARNING|(flags)), \ "i" (sizeof(struct bug_entry))); \ } while (0) diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index cecf559d0012..39e702d90cdb 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -76,7 +76,7 @@ do { \ unreachable(); \ } while (0) -#define __WARN_TAINT(taint) _BUG_FLAGS(ASM_UD0, BUGFLAG_TAINT(taint)) +#define __WARN_FLAGS(flags) _BUG_FLAGS(ASM_UD0, BUGFLAG_WARNING|(flags)) #include diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index 6f96247226a4..5fe9f39cec44 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -5,6 +5,8 @@ #ifdef CONFIG_GENERIC_BUG #define BUGFLAG_WARNING (1 << 0) +#define BUGFLAG_ONCE (1 << 1) +#define BUGFLAG_DONE (1 << 2) #define BUGFLAG_TAINT(taint) (BUGFLAG_WARNING | ((taint) << 8)) #define BUG_GET_TAINT(bug) ((bug)->flags >> 8) #endif @@ -55,6 +57,18 @@ struct bug_entry { #define BUG_ON(condition) do { if (unlikely(condition)) BUG(); } while (0) #endif +#ifdef __WARN_FLAGS +#define __WARN_TAINT(taint) __WARN_FLAGS(BUGFLAG_TAINT(taint)) +#define __WARN_ONCE_TAINT(taint) __WARN_FLAGS(BUGFLAG_ONCE|BUGFLAG_TAINT(taint)) + +#define WARN_ON_ONCE(condition) ({ \ + int __ret_warn_on = !!(condition); \ + if (unlikely(__ret_warn_on)) \ + __WARN_ONCE_TAINT(TAINT_WARN); \ + unlikely(__ret_warn_on); \ +}) +#endif + /* * WARN(), WARN_ON(), WARN_ON_ONCE, and so on can be used to report * significant issues that need prompt attention if they should ever @@ -97,7 +111,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, #endif #ifndef WARN -#define WARN(condition, format...) ({ \ +#define WARN(condition, format...) ({ \ int __ret_warn_on = !!(condition); \ if (unlikely(__ret_warn_on)) \ __WARN_printf(format); \ @@ -112,6 +126,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, unlikely(__ret_warn_on); \ }) +#ifndef WARN_ON_ONCE #define WARN_ON_ONCE(condition) ({ \ static bool __section(.data.unlikely) __warned; \ int __ret_warn_once = !!(condition); \ @@ -122,6 +137,7 @@ void __warn(const char *file, int line, void *caller, unsigned taint, } \ unlikely(__ret_warn_once); \ }) +#endif #define WARN_ONCE(condition, format...) ({ \ static bool __section(.data.unlikely) __warned; \ diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 0968d13b3885..51c8dbe8e8d9 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -286,8 +286,6 @@ *(.rodata1) \ } \ \ - BUG_TABLE \ - \ /* PCI quirks */ \ .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ @@ -855,7 +853,8 @@ READ_MOSTLY_DATA(cacheline) \ DATA_DATA \ CONSTRUCTORS \ - } + } \ + BUG_TABLE #define INIT_TEXT_SECTION(inittext_align) \ . = ALIGN(inittext_align); \ diff --git a/include/linux/bug.h b/include/linux/bug.h index 5828489309bb..687b557fc5eb 100644 --- a/include/linux/bug.h +++ b/include/linux/bug.h @@ -105,7 +105,7 @@ static inline int is_warning_bug(const struct bug_entry *bug) return bug->flags & BUGFLAG_WARNING; } -const struct bug_entry *find_bug(unsigned long bugaddr); +struct bug_entry *find_bug(unsigned long bugaddr); enum bug_trap_type report_bug(unsigned long bug_addr, struct pt_regs *regs); diff --git a/lib/bug.c b/lib/bug.c index 06edbbef0623..a6a1137d06db 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -47,7 +47,7 @@ #include #include -extern const struct bug_entry __start___bug_table[], __stop___bug_table[]; +extern struct bug_entry __start___bug_table[], __stop___bug_table[]; static inline unsigned long bug_addr(const struct bug_entry *bug) { @@ -62,10 +62,10 @@ static inline unsigned long bug_addr(const struct bug_entry *bug) /* Updates are protected by module mutex */ static LIST_HEAD(module_bug_list); -static const struct bug_entry *module_find_bug(unsigned long bugaddr) +static struct bug_entry *module_find_bug(unsigned long bugaddr) { struct module *mod; - const struct bug_entry *bug = NULL; + struct bug_entry *bug = NULL; rcu_read_lock_sched(); list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { @@ -122,15 +122,15 @@ void module_bug_cleanup(struct module *mod) #else -static inline const struct bug_entry *module_find_bug(unsigned long bugaddr) +static inline struct bug_entry *module_find_bug(unsigned long bugaddr) { return NULL; } #endif -const struct bug_entry *find_bug(unsigned long bugaddr) +struct bug_entry *find_bug(unsigned long bugaddr) { - const struct bug_entry *bug; + struct bug_entry *bug; for (bug = __start___bug_table; bug < __stop___bug_table; ++bug) if (bugaddr == bug_addr(bug)) @@ -141,9 +141,9 @@ const struct bug_entry *find_bug(unsigned long bugaddr) enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) { - const struct bug_entry *bug; + struct bug_entry *bug; const char *file; - unsigned line, warning; + unsigned line, warning, once, done; if (!is_valid_bugaddr(bugaddr)) return BUG_TRAP_TYPE_NONE; @@ -164,6 +164,18 @@ enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs) line = bug->line; #endif warning = (bug->flags & BUGFLAG_WARNING) != 0; + once = (bug->flags & BUGFLAG_ONCE) != 0; + done = (bug->flags & BUGFLAG_DONE) != 0; + + if (warning && once) { + if (done) + return BUG_TRAP_TYPE_WARN; + + /* + * Since this is the only store, concurrency is not an issue. + */ + bug->flags |= BUGFLAG_DONE; + } } if (warning) { -- cgit v1.2.3 From 6bcdf9d51b9892dd5c6892d50cf5e9628efb8062 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2017 13:47:11 -0500 Subject: x86/unwind: Read stack return address in update_stack_state() Instead of reading the return address when unwind_get_return_address() is called, read it from update_stack_state() and store it in the unwind state. This enables the next patch to check the return address from unwind_next_frame() so it can detect an entry code frame. Signed-off-by: Josh Poimboeuf Acked-by: Thomas Gleixner Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Brian Gerst Cc: Daniel Borkmann Cc: Dave Jones Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/af0c5e4560c49c0343dca486ea26c4fa92bc4e35.1492020577.git.jpoimboe@redhat.com Signed-off-by: Ingo Molnar --- arch/x86/include/asm/unwind.h | 1 + arch/x86/kernel/unwind_frame.c | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'arch/x86/include/asm') diff --git a/arch/x86/include/asm/unwind.h b/arch/x86/include/asm/unwind.h index 6fa75b17aec3..5663425f812f 100644 --- a/arch/x86/include/asm/unwind.h +++ b/arch/x86/include/asm/unwind.h @@ -14,6 +14,7 @@ struct unwind_state { #ifdef CONFIG_FRAME_POINTER unsigned long *bp, *orig_sp; struct pt_regs *regs; + unsigned long ip; #else unsigned long *sp; #endif diff --git a/arch/x86/kernel/unwind_frame.c b/arch/x86/kernel/unwind_frame.c index 9098ef1da8f8..c67a059c6655 100644 --- a/arch/x86/kernel/unwind_frame.c +++ b/arch/x86/kernel/unwind_frame.c @@ -56,20 +56,10 @@ static void unwind_dump(struct unwind_state *state, unsigned long *sp) unsigned long unwind_get_return_address(struct unwind_state *state) { - unsigned long addr; - unsigned long *addr_p = unwind_get_return_address_ptr(state); - if (unwind_done(state)) return 0; - if (state->regs && user_mode(state->regs)) - return 0; - - addr = READ_ONCE_TASK_STACK(state->task, *addr_p); - addr = ftrace_graph_ret_addr(state->task, &state->graph_idx, addr, - addr_p); - - return __kernel_text_address(addr) ? addr : 0; + return __kernel_text_address(state->ip) ? state->ip : 0; } EXPORT_SYMBOL_GPL(unwind_get_return_address); @@ -141,7 +131,7 @@ static bool update_stack_state(struct unwind_state *state, struct stack_info *info = &state->stack_info; enum stack_type prev_type = info->type; struct pt_regs *regs; - unsigned long *frame, *prev_frame_end; + unsigned long *frame, *prev_frame_end, *addr_p, addr; size_t len; if (state->regs) @@ -185,6 +175,16 @@ static bool update_stack_state(struct unwind_state *state, state->regs = NULL; } + /* Save the return address: */ + if (state->regs && user_mode(state->regs)) + state->ip = 0; + else { + addr_p = unwind_get_return_address_ptr(state); + addr = READ_ONCE_TASK_STACK(state->task, *addr_p); + state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, + addr, addr_p); + } + /* Save the original stack pointer for unwind_dump(): */ if (!state->orig_sp || info->type != prev_type) state->orig_sp = frame; @@ -228,6 +228,7 @@ bool unwind_next_frame(struct unwind_state *state) */ state->regs = regs; state->bp = NULL; + state->ip = 0; return true; } -- cgit v1.2.3 From a8b7a92318b6d7779f6d8e9aa6ba0e3de01a8943 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Wed, 12 Apr 2017 13:47:12 -0500 Subject: x86/unwind: Silence entry-related warnings A few people have reported unwinder warnings like the following: WARNING: kernel stack frame pointer at ffffc90000fe7ff0 in rsync:1157 has bad value (null) unwind stack type:0 next_sp: (null) mask:2 graph_idx:0 ffffc90000fe7f98: ffffc90000fe7ff0 (0xffffc90000fe7ff0) ffffc90000fe7fa0: ffffffffb7000f56 (trace_hardirqs_off_thunk+0x1a/0x1c) ffffc90000fe7fa8: 0000000000000246 (0x246) ffffc90000fe7fb0: 0000000000000000 ... ffffc90000fe7fc0: 00007ffe3af639bc (0x7ffe3af639bc) ffffc90000fe7fc8: 0000000000000006 (0x6) ffffc90000fe7fd0: 00007f80af433fc5 (0x7f80af433fc5) ffffc90000fe7fd8: 00007ffe3af638e0 (0x7ffe3af638e0) ffffc90000fe7fe0: 00007ffe3af638e0 (0x7ffe3af638e0) ffffc90000fe7fe8: 00007ffe3af63970 (0x7ffe3af63970) ffffc90000fe7ff0: 0000000000000000 ... ffffc90000fe7ff8: ffffffffb7b74b9a (entry_SYSCALL_64_after_swapgs+0x17/0x4f) This warning can happen when unwinding a code path where an interrupt occurred in x86 entry code before it set up the first stack frame. Silently ignore any warnings for this case. Reported-by: Daniel Borkmann Reported-by: Dave Jones Signed-off-by: Josh Poimboeuf Acked-by: Thomas Gleixner Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Brian Gerst Cc: Denys Vlasenko Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Peter Zijlstra Fixes: c32c47c68a0a ("x86/unwind: Warn on bad frame pointer") Link: http://lkml.kernel.org/r/dbd6838826466a60dc23a52098185bc973ce2f1e.1492020577.git.jpoimboe@redhat.com Signed-off-by: Ingo Molnar --- arch/x86/include/asm/unwind.h | 1 + arch/x86/kernel/unwind_frame.c | 36 +++++++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) (limited to 'arch/x86/include/asm') diff --git a/arch/x86/include/asm/unwind.h b/arch/x86/include/asm/unwind.h index 5663425f812f..9b10dcd51716 100644 --- a/arch/x86/include/asm/unwind.h +++ b/arch/x86/include/asm/unwind.h @@ -12,6 +12,7 @@ struct unwind_state { struct task_struct *task; int graph_idx; #ifdef CONFIG_FRAME_POINTER + bool got_irq; unsigned long *bp, *orig_sp; struct pt_regs *regs; unsigned long ip; diff --git a/arch/x86/kernel/unwind_frame.c b/arch/x86/kernel/unwind_frame.c index c67a059c6655..c4611359b617 100644 --- a/arch/x86/kernel/unwind_frame.c +++ b/arch/x86/kernel/unwind_frame.c @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -72,6 +74,21 @@ static size_t regs_size(struct pt_regs *regs) return sizeof(*regs); } +static bool in_entry_code(unsigned long ip) +{ + char *addr = (char *)ip; + + if (addr >= __entry_text_start && addr < __entry_text_end) + return true; + +#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN) + if (addr >= __irqentry_text_start && addr < __irqentry_text_end) + return true; +#endif + + return false; +} + #ifdef CONFIG_X86_32 #define GCC_REALIGN_WORDS 3 #else @@ -144,6 +161,7 @@ static bool update_stack_state(struct unwind_state *state, if (regs) { frame = (unsigned long *)regs; len = regs_size(regs); + state->got_irq = true; } else { frame = next_bp; len = FRAME_HEADER_SIZE; @@ -239,16 +257,8 @@ bool unwind_next_frame(struct unwind_state *state) next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp); /* Move to the next frame if it's safe: */ - if (!update_stack_state(state, next_bp)) { - /* - * Don't warn on bad regs->bp. An interrupt in entry code - * might cause a false positive warning. - */ - if (state->regs) - goto the_end; - + if (!update_stack_state(state, next_bp)) goto bad_address; - } return true; @@ -263,6 +273,13 @@ bad_address: if (state->task != current) goto the_end; + /* + * Don't warn if the unwinder got lost due to an interrupt in entry + * code before the stack was set up: + */ + if (state->got_irq && in_entry_code(state->ip)) + goto the_end; + if (state->regs) { printk_deferred_once(KERN_WARNING "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n", @@ -289,6 +306,7 @@ void __unwind_start(struct unwind_state *state, struct task_struct *task, memset(state, 0, sizeof(*state)); state->task = task; + state->got_irq = (regs); /* Don't even attempt to start from user mode regs: */ if (regs && user_mode(regs)) { -- cgit v1.2.3