From f94909ceb1ed4bfdb2ada72f93236305e6d6951f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Sat, 4 Dec 2021 14:43:40 +0100 Subject: x86: Prepare asm files for straight-line-speculation Replace all ret/retq instructions with RET in preparation of making RET a macro. Since AS is case insensitive it's a big no-op without RET defined. find arch/x86/ -name \*.S | while read file do sed -i 's/\/RET/' $file done Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov Link: https://lore.kernel.org/r/20211204134907.905503893@infradead.org --- arch/x86/entry/entry_32.S | 2 +- arch/x86/entry/entry_64.S | 10 +++++----- arch/x86/entry/thunk_32.S | 2 +- arch/x86/entry/thunk_64.S | 2 +- arch/x86/entry/vdso/vdso32/system_call.S | 2 +- arch/x86/entry/vdso/vsgx.S | 2 +- arch/x86/entry/vsyscall/vsyscall_emu_64.S | 6 +++--- 7 files changed, 13 insertions(+), 13 deletions(-) (limited to 'arch/x86/entry') diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index ccb9d32768f3..00413e37feee 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -740,7 +740,7 @@ SYM_FUNC_START(schedule_tail_wrapper) popl %eax FRAME_END - ret + RET SYM_FUNC_END(schedule_tail_wrapper) .popsection diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 97b1f84bb53f..e23319ad3f42 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -738,7 +738,7 @@ SYM_FUNC_START(asm_load_gs_index) 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE swapgs FRAME_END - ret + RET SYM_FUNC_END(asm_load_gs_index) EXPORT_SYMBOL(asm_load_gs_index) @@ -889,7 +889,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) * is needed here. */ SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx - ret + RET .Lparanoid_entry_checkgs: /* EBX = 1 -> kernel GSBASE active, no restore required */ @@ -910,7 +910,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) .Lparanoid_kernel_gsbase: FENCE_SWAPGS_KERNEL_ENTRY - ret + RET SYM_CODE_END(paranoid_entry) /* @@ -989,7 +989,7 @@ SYM_CODE_START_LOCAL(error_entry) movq %rax, %rsp /* switch stack */ ENCODE_FRAME_POINTER pushq %r12 - ret + RET /* * There are two places in the kernel that can potentially fault with @@ -1020,7 +1020,7 @@ SYM_CODE_START_LOCAL(error_entry) */ .Lerror_entry_done_lfence: FENCE_SWAPGS_KERNEL_ENTRY - ret + RET .Lbstep_iret: /* Fix truncated RIP */ diff --git a/arch/x86/entry/thunk_32.S b/arch/x86/entry/thunk_32.S index f1f96d4d8cd6..7591bab060f7 100644 --- a/arch/x86/entry/thunk_32.S +++ b/arch/x86/entry/thunk_32.S @@ -24,7 +24,7 @@ SYM_CODE_START_NOALIGN(\name) popl %edx popl %ecx popl %eax - ret + RET _ASM_NOKPROBE(\name) SYM_CODE_END(\name) .endm diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S index 496b11ec469d..505b488fcc65 100644 --- a/arch/x86/entry/thunk_64.S +++ b/arch/x86/entry/thunk_64.S @@ -50,7 +50,7 @@ SYM_CODE_START_LOCAL_NOALIGN(__thunk_restore) popq %rsi popq %rdi popq %rbp - ret + RET _ASM_NOKPROBE(__thunk_restore) SYM_CODE_END(__thunk_restore) #endif diff --git a/arch/x86/entry/vdso/vdso32/system_call.S b/arch/x86/entry/vdso/vdso32/system_call.S index 6ddd7a937b3e..d33c6513fd2c 100644 --- a/arch/x86/entry/vdso/vdso32/system_call.S +++ b/arch/x86/entry/vdso/vdso32/system_call.S @@ -78,7 +78,7 @@ SYM_INNER_LABEL(int80_landing_pad, SYM_L_GLOBAL) popl %ecx CFI_RESTORE ecx CFI_ADJUST_CFA_OFFSET -4 - ret + RET CFI_ENDPROC .size __kernel_vsyscall,.-__kernel_vsyscall diff --git a/arch/x86/entry/vdso/vsgx.S b/arch/x86/entry/vdso/vsgx.S index 99dafac992e2..d77d278ee9dd 100644 --- a/arch/x86/entry/vdso/vsgx.S +++ b/arch/x86/entry/vdso/vsgx.S @@ -81,7 +81,7 @@ SYM_FUNC_START(__vdso_sgx_enter_enclave) pop %rbx leave .cfi_def_cfa %rsp, 8 - ret + RET /* The out-of-line code runs with the pre-leave stack frame. */ .cfi_def_cfa %rbp, 16 diff --git a/arch/x86/entry/vsyscall/vsyscall_emu_64.S b/arch/x86/entry/vsyscall/vsyscall_emu_64.S index 2e203f3a25a7..15e35159ebb6 100644 --- a/arch/x86/entry/vsyscall/vsyscall_emu_64.S +++ b/arch/x86/entry/vsyscall/vsyscall_emu_64.S @@ -19,17 +19,17 @@ __vsyscall_page: mov $__NR_gettimeofday, %rax syscall - ret + RET .balign 1024, 0xcc mov $__NR_time, %rax syscall - ret + RET .balign 1024, 0xcc mov $__NR_getcpu, %rax syscall - ret + RET .balign 4096, 0xcc -- cgit v1.2.3 From 16e617d05ef0c521d000c989796412ce713f28c9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 10 Nov 2021 11:01:07 +0100 Subject: x86/entry_64: Remove .fixup usage Place the anonymous .fixup code at the tail of the regular functions. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Josh Poimboeuf Reviewed-by: Borislav Petkov Reviewed-by: Lai Jiangshan Link: https://lore.kernel.org/r/20211110101325.186049322@infradead.org --- arch/x86/entry/entry_64.S | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'arch/x86/entry') diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index e23319ad3f42..1ffdbfaad2e2 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -739,13 +739,9 @@ SYM_FUNC_START(asm_load_gs_index) swapgs FRAME_END RET -SYM_FUNC_END(asm_load_gs_index) -EXPORT_SYMBOL(asm_load_gs_index) - _ASM_EXTABLE(.Lgs_change, .Lbad_gs) - .section .fixup, "ax" /* running with kernelgs */ -SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) +.Lbad_gs: swapgs /* switch back to user gs */ .macro ZAP_GS /* This can't be a string because the preprocessor needs to see it. */ @@ -756,8 +752,11 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) xorl %eax, %eax movl %eax, %gs jmp 2b -SYM_CODE_END(.Lbad_gs) - .previous + + _ASM_EXTABLE(.Lgs_change, .Lbad_gs) + +SYM_FUNC_END(asm_load_gs_index) +EXPORT_SYMBOL(asm_load_gs_index) #ifdef CONFIG_XEN_PV /* -- cgit v1.2.3 From aa93e2ad7464ffb90155a5ffdde963816f86d5dc Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 10 Nov 2021 11:01:08 +0100 Subject: x86/entry_32: Remove .fixup usage Where possible, push the .fixup into code, at the tail of functions. This is hard for macros since they're used in multiple functions, therefore introduce a new extable handler to pop zeros. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/r/20211110101325.245184699@infradead.org --- arch/x86/entry/entry_32.S | 28 ++++++++-------------------- arch/x86/include/asm/extable_fixup_types.h | 2 ++ arch/x86/mm/extable.c | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 20 deletions(-) (limited to 'arch/x86/entry') diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index 00413e37feee..e0a95d8a6553 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -270,17 +270,9 @@ 3: popl %fs addl $(4 + \pop), %esp /* pop the unused "gs" slot */ IRET_FRAME -.pushsection .fixup, "ax" -4: movl $0, (%esp) - jmp 1b -5: movl $0, (%esp) - jmp 2b -6: movl $0, (%esp) - jmp 3b -.popsection - _ASM_EXTABLE(1b, 4b) - _ASM_EXTABLE(2b, 5b) - _ASM_EXTABLE(3b, 6b) + _ASM_EXTABLE_TYPE(1b, 1b, EX_TYPE_POP_ZERO) + _ASM_EXTABLE_TYPE(2b, 2b, EX_TYPE_POP_ZERO) + _ASM_EXTABLE_TYPE(3b, 3b, EX_TYPE_POP_ZERO) .endm .macro RESTORE_ALL_NMI cr3_reg:req pop=0 @@ -925,10 +917,8 @@ SYM_FUNC_START(entry_SYSENTER_32) sti sysexit -.pushsection .fixup, "ax" -2: movl $0, PT_FS(%esp) - jmp 1b -.popsection +2: movl $0, PT_FS(%esp) + jmp 1b _ASM_EXTABLE(1b, 2b) .Lsysenter_fix_flags: @@ -996,8 +986,7 @@ restore_all_switch_stack: */ iret -.section .fixup, "ax" -SYM_CODE_START(asm_iret_error) +.Lasm_iret_error: pushl $0 # no error code pushl $iret_error @@ -1014,9 +1003,8 @@ SYM_CODE_START(asm_iret_error) #endif jmp handle_exception -SYM_CODE_END(asm_iret_error) -.previous - _ASM_EXTABLE(.Lirq_return, asm_iret_error) + + _ASM_EXTABLE(.Lirq_return, .Lasm_iret_error) SYM_FUNC_END(entry_INT80_32) .macro FIXUP_ESPFIX_STACK diff --git a/arch/x86/include/asm/extable_fixup_types.h b/arch/x86/include/asm/extable_fixup_types.h index 409524d5d2eb..4d709a2768bb 100644 --- a/arch/x86/include/asm/extable_fixup_types.h +++ b/arch/x86/include/asm/extable_fixup_types.h @@ -19,4 +19,6 @@ #define EX_TYPE_DEFAULT_MCE_SAFE 12 #define EX_TYPE_FAULT_MCE_SAFE 13 +#define EX_TYPE_POP_ZERO 14 + #endif diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 5cd2a88930a9..fb0c4752df1a 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -99,6 +99,18 @@ static bool ex_handler_clear_fs(const struct exception_table_entry *fixup, return ex_handler_default(fixup, regs); } +static bool ex_handler_pop_zero(const struct exception_table_entry *fixup, + struct pt_regs *regs) +{ + /* + * Typically used for when "pop %seg" traps, in which case we'll clear + * the stack slot and re-try the instruction, which will then succeed + * to pop zero. + */ + *((unsigned long *)regs->sp) = 0; + return ex_handler_default(fixup, regs); +} + int ex_get_fixup_type(unsigned long ip) { const struct exception_table_entry *e = search_exception_tables(ip); @@ -156,6 +168,8 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, case EX_TYPE_WRMSR_IN_MCE: ex_handler_msr_mce(regs, true); break; + case EX_TYPE_POP_ZERO: + return ex_handler_pop_zero(e, regs); } BUG(); } -- cgit v1.2.3 From e5eefda5aa51f3178821b58806e1dddd798c0934 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 10 Nov 2021 11:01:24 +0100 Subject: x86: Remove .fixup section No moar users, kill it dead. Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Josh Poimboeuf Link: https://lore.kernel.org/r/20211110101326.201590122@infradead.org --- arch/x86/entry/vdso/vdso-layout.lds.S | 1 - arch/x86/kernel/vmlinux.lds.S | 1 - 2 files changed, 2 deletions(-) (limited to 'arch/x86/entry') diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S index dc8da7695859..bafa73f09e92 100644 --- a/arch/x86/entry/vdso/vdso-layout.lds.S +++ b/arch/x86/entry/vdso/vdso-layout.lds.S @@ -77,7 +77,6 @@ SECTIONS .text : { *(.text*) - *(.fixup) } :text =0x90909090, diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 3d6dc12d198f..27f830345b6f 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -137,7 +137,6 @@ SECTIONS ALIGN_ENTRY_TEXT_END SOFTIRQENTRY_TEXT STATIC_CALL_TEXT - *(.fixup) *(.gnu.warning) #ifdef CONFIG_RETPOLINE -- cgit v1.2.3 From 9cdbeec4096804083944d05da96bbaf59a1eb4f9 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 11 Jan 2022 12:11:14 +0100 Subject: x86/entry_32: Fix segment exceptions The LKP robot reported that commit in Fixes: caused a failure. Turns out the ldt_gdt_32 selftest turns into an infinite loop trying to clear the segment. As discovered by Sean, what happens is that PARANOID_EXIT_TO_KERNEL_MODE in the handle_exception_return path overwrites the entry stack data with the task stack data, restoring the "bad" segment value. Instead of having the exception retry the instruction, have it emulate the full instruction. Replace EX_TYPE_POP_ZERO with EX_TYPE_POP_REG which will do the equivalent of: POP %reg; MOV $imm, %reg. In order to encode the segment registers, add them as registers 8-11 for 32-bit. By setting regs->[defg]s the (nested) RESTORE_REGS will pop this value at the end of the exception handler and by increasing regs->sp, it will have skipped the stack slot. This was debugged by Sean Christopherson . [ bp: Add EX_REG_GS too. ] Fixes: aa93e2ad7464 ("x86/entry_32: Remove .fixup usage") Reported-by: kernel test robot Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Borislav Petkov Link: https://lore.kernel.org/r/Yd1l0gInc4zRcnt/@hirez.programming.kicks-ass.net --- arch/x86/entry/entry_32.S | 13 +++++++++---- arch/x86/include/asm/extable_fixup_types.h | 11 ++++++++++- arch/x86/lib/insn-eval.c | 5 +++++ arch/x86/mm/extable.c | 17 +++-------------- 4 files changed, 27 insertions(+), 19 deletions(-) (limited to 'arch/x86/entry') diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index e0a95d8a6553..a7ec22b1d06c 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -268,11 +268,16 @@ 1: popl %ds 2: popl %es 3: popl %fs - addl $(4 + \pop), %esp /* pop the unused "gs" slot */ +4: addl $(4 + \pop), %esp /* pop the unused "gs" slot */ IRET_FRAME - _ASM_EXTABLE_TYPE(1b, 1b, EX_TYPE_POP_ZERO) - _ASM_EXTABLE_TYPE(2b, 2b, EX_TYPE_POP_ZERO) - _ASM_EXTABLE_TYPE(3b, 3b, EX_TYPE_POP_ZERO) + + /* + * There is no _ASM_EXTABLE_TYPE_REG() for ASM, however since this is + * ASM the registers are known and we can trivially hard-code them. + */ + _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_POP_ZERO|EX_REG_DS) + _ASM_EXTABLE_TYPE(2b, 3b, EX_TYPE_POP_ZERO|EX_REG_ES) + _ASM_EXTABLE_TYPE(3b, 4b, EX_TYPE_POP_ZERO|EX_REG_FS) .endm .macro RESTORE_ALL_NMI cr3_reg:req pop=0 diff --git a/arch/x86/include/asm/extable_fixup_types.h b/arch/x86/include/asm/extable_fixup_types.h index b5ab333e064a..503622627400 100644 --- a/arch/x86/include/asm/extable_fixup_types.h +++ b/arch/x86/include/asm/extable_fixup_types.h @@ -16,9 +16,16 @@ #define EX_DATA_FLAG_SHIFT 12 #define EX_DATA_IMM_SHIFT 16 +#define EX_DATA_REG(reg) ((reg) << EX_DATA_REG_SHIFT) #define EX_DATA_FLAG(flag) ((flag) << EX_DATA_FLAG_SHIFT) #define EX_DATA_IMM(imm) ((imm) << EX_DATA_IMM_SHIFT) +/* segment regs */ +#define EX_REG_DS EX_DATA_REG(8) +#define EX_REG_ES EX_DATA_REG(9) +#define EX_REG_FS EX_DATA_REG(10) +#define EX_REG_GS EX_DATA_REG(11) + /* flags */ #define EX_FLAG_CLEAR_AX EX_DATA_FLAG(1) #define EX_FLAG_CLEAR_DX EX_DATA_FLAG(2) @@ -41,7 +48,9 @@ #define EX_TYPE_RDMSR_IN_MCE 13 #define EX_TYPE_DEFAULT_MCE_SAFE 14 #define EX_TYPE_FAULT_MCE_SAFE 15 -#define EX_TYPE_POP_ZERO 16 + +#define EX_TYPE_POP_REG 16 /* sp += sizeof(long) */ +#define EX_TYPE_POP_ZERO (EX_TYPE_POP_REG | EX_DATA_IMM(0)) #define EX_TYPE_IMM_REG 17 /* reg := (long)imm */ #define EX_TYPE_EFAULT_REG (EX_TYPE_IMM_REG | EX_DATA_IMM(-EFAULT)) diff --git a/arch/x86/lib/insn-eval.c b/arch/x86/lib/insn-eval.c index 7760d228041b..c8a962c2e653 100644 --- a/arch/x86/lib/insn-eval.c +++ b/arch/x86/lib/insn-eval.c @@ -430,6 +430,11 @@ static const int pt_regoff[] = { offsetof(struct pt_regs, r13), offsetof(struct pt_regs, r14), offsetof(struct pt_regs, r15), +#else + offsetof(struct pt_regs, ds), + offsetof(struct pt_regs, es), + offsetof(struct pt_regs, fs), + offsetof(struct pt_regs, gs), #endif }; diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 41eaa648349e..dba2197c05c3 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -126,18 +126,6 @@ static bool ex_handler_clear_fs(const struct exception_table_entry *fixup, return ex_handler_default(fixup, regs); } -static bool ex_handler_pop_zero(const struct exception_table_entry *fixup, - struct pt_regs *regs) -{ - /* - * Typically used for when "pop %seg" traps, in which case we'll clear - * the stack slot and re-try the instruction, which will then succeed - * to pop zero. - */ - *((unsigned long *)regs->sp) = 0; - return ex_handler_default(fixup, regs); -} - static bool ex_handler_imm_reg(const struct exception_table_entry *fixup, struct pt_regs *regs, int reg, int imm) { @@ -218,8 +206,9 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, case EX_TYPE_RDMSR_IN_MCE: ex_handler_msr_mce(regs, false); break; - case EX_TYPE_POP_ZERO: - return ex_handler_pop_zero(e, regs); + case EX_TYPE_POP_REG: + regs->sp += sizeof(long); + fallthrough; case EX_TYPE_IMM_REG: return ex_handler_imm_reg(e, regs, reg, imm); case EX_TYPE_FAULT_SGX: -- cgit v1.2.3