From c3c7f14a116c24d6fba185c95cd7454f3764f8a9 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 26 Jan 2012 18:06:45 -0500 Subject: x86/jump-label: Use best default nops for inital jump label calls As specified by H. Peter Anvin, the best nops for x86 without knowing the running computer is: 32bit: 0x3e, 0x8d, 0x74, 0x26, 0x00 also known as GENERIC_NOP5_ATOMIC 64bit: 0x0f, 0x1f, 0x44, 0x00, 0x00 also known as P6_NOP5_ATOMIC Currently the default nop that is used by jump label is: 0xe9 0x00 0x00 0x00 0x00 Which is really a 5byte jump to the next position. It's better to use a real nop than a jmp. Cc: H. Peter Anvin Cc: Jason Baron Signed-off-by: Steven Rostedt --- arch/x86/include/asm/jump_label.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index 3a16c1483b45..64507f35800c 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -3,18 +3,23 @@ #ifdef __KERNEL__ +#include #include #include #include #define JUMP_LABEL_NOP_SIZE 5 -#define STATIC_KEY_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t" +#ifdef CONFIG_X86_64 +# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC +#else +# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC +#endif static __always_inline bool arch_static_branch(struct static_key *key) { asm goto("1:" - STATIC_KEY_INITIAL_NOP + ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" ".pushsection __jump_table, \"aw\" \n\t" _ASM_ALIGN "\n\t" _ASM_PTR "1b, %l[l_yes], %c0 \n\t" -- cgit v1.2.3 From 11570da1c5b1dee1dc1439159e967ee44214a78b Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 26 Jan 2012 18:16:15 -0500 Subject: x86/jump-label: Do not bother updating nops if they are correct On boot up, the jump label init function scans all the jump label locations and converts them to the best nop for the machine. If the nop is already the ideal nop, do not bother with changing it. Cc: Jason Baron Cc: H. Peter Anvin Signed-off-by: Steven Rostedt --- arch/x86/kernel/jump_label.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 2889b3d43882..ae3d8fb8039f 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -50,10 +50,33 @@ void arch_jump_label_transform(struct jump_entry *entry, put_online_cpus(); } +static enum { + JL_STATE_START, + JL_STATE_NO_UPDATE, + JL_STATE_UPDATE, +} jlstate __initdata_or_module = JL_STATE_START; + __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type) { - __jump_label_transform(entry, type, text_poke_early); + /* + * This function is called at boot up and when modules are + * first loaded. Check if the default nop, the one that is + * inserted at compile time, is the ideal nop. If it is, then + * we do not need to update the nop, and we can leave it as is. + * If it is not, then we need to update the nop to the ideal nop. + */ + if (jlstate == JL_STATE_START) { + const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; + const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; + + if (memcmp(ideal_nop, default_nop, 5) != 0) + jlstate = JL_STATE_UPDATE; + else + jlstate = JL_STATE_NO_UPDATE; + } + if (jlstate == JL_STATE_UPDATE) + __jump_label_transform(entry, type, text_poke_early); } #endif -- cgit v1.2.3 From 9c85f3bdf400665eecf62658a9106501f6a77a13 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Thu, 26 Jan 2012 18:38:07 -0500 Subject: x86/jump-label: Add safety checks to jump label conversions As with all modifying of kernel text, we need to be very paranoid. When converting the jump label locations to and from nops to jumps a check has been added to make sure what we are replacing is what we expect, otherwise we bug. Cc: H. Peter Anvin Cc: Jason Baron Signed-off-by: Steven Rostedt --- arch/x86/kernel/jump_label.c | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index ae3d8fb8039f..24cf2b25ce73 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -26,16 +26,40 @@ union jump_code_union { static void __jump_label_transform(struct jump_entry *entry, enum jump_label_type type, - void *(*poker)(void *, const void *, size_t)) + void *(*poker)(void *, const void *, size_t), + int init) { union jump_code_union code; + const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; if (type == JUMP_LABEL_ENABLE) { + /* + * We are enabling this jump label. If it is not a nop + * then something must have gone wrong. + */ + BUG_ON(memcmp((void *)entry->code, ideal_nop, 5) != 0); + code.jump = 0xe9; code.offset = entry->target - (entry->code + JUMP_LABEL_NOP_SIZE); - } else + } else { + /* + * We are disabling this jump label. If it is not what + * we think it is, then something must have gone wrong. + * If this is the first initialization call, then we + * are converting the default nop to the ideal nop. + */ + if (init) { + const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; + BUG_ON(memcmp((void *)entry->code, default_nop, 5) != 0); + } else { + code.jump = 0xe9; + code.offset = entry->target - + (entry->code + JUMP_LABEL_NOP_SIZE); + BUG_ON(memcmp((void *)entry->code, &code, 5) != 0); + } memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); + } (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); } @@ -45,7 +69,7 @@ void arch_jump_label_transform(struct jump_entry *entry, { get_online_cpus(); mutex_lock(&text_mutex); - __jump_label_transform(entry, type, text_poke_smp); + __jump_label_transform(entry, type, text_poke_smp, 0); mutex_unlock(&text_mutex); put_online_cpus(); } @@ -76,7 +100,7 @@ __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry, jlstate = JL_STATE_NO_UPDATE; } if (jlstate == JL_STATE_UPDATE) - __jump_label_transform(entry, type, text_poke_early); + __jump_label_transform(entry, type, text_poke_early, 1); } #endif -- cgit v1.2.3 From fb40d7a8994a3cc7a1e1c1f3258ea8662a366916 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 1 Feb 2012 09:59:24 -0500 Subject: x86/jump-label: Show where and what was wrong on errors When modifying text sections for jump labels, a paranoid check is performed. If the check fails, the system "bugs". But why it failed is not shown. The BUG_ON()s in the jump label update code is replaced with bug_at(ip). This is a function that will show what pointer failed, and what was at the location of the failure that made jump label panic. Signed-off-by: Steven Rostedt --- arch/x86/kernel/jump_label.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 24cf2b25ce73..912a52812916 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -24,6 +24,18 @@ union jump_code_union { } __attribute__((packed)); }; +static void bug_at(unsigned char *ip, int line) +{ + /* + * The location is not an op that we were expecting. + * Something went wrong. Crash the box, as something could be + * corrupting the kernel. + */ + pr_warning("Unexpected op at %pS [%p] (%02x %02x %02x %02x %02x) %s:%d\n", + ip, ip, ip[0], ip[1], ip[2], ip[3], ip[4], __FILE__, line); + BUG(); +} + static void __jump_label_transform(struct jump_entry *entry, enum jump_label_type type, void *(*poker)(void *, const void *, size_t), @@ -37,7 +49,8 @@ static void __jump_label_transform(struct jump_entry *entry, * We are enabling this jump label. If it is not a nop * then something must have gone wrong. */ - BUG_ON(memcmp((void *)entry->code, ideal_nop, 5) != 0); + if (unlikely(memcmp((void *)entry->code, ideal_nop, 5) != 0)) + bug_at((void *)entry->code, __LINE__); code.jump = 0xe9; code.offset = entry->target - @@ -51,12 +64,14 @@ static void __jump_label_transform(struct jump_entry *entry, */ if (init) { const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; - BUG_ON(memcmp((void *)entry->code, default_nop, 5) != 0); + if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0)) + bug_at((void *)entry->code, __LINE__); } else { code.jump = 0xe9; code.offset = entry->target - (entry->code + JUMP_LABEL_NOP_SIZE); - BUG_ON(memcmp((void *)entry->code, &code, 5) != 0); + if (unlikely(memcmp((void *)entry->code, &code, 5) != 0)) + bug_at((void *)entry->code, __LINE__); } memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); } -- cgit v1.2.3