From b89d82ef01b33bc50cbaa8ff05607879b40d0704 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 8 Jan 2019 16:19:01 +0000 Subject: arm64: kpti: Avoid rewriting early page tables when KASLR is enabled A side effect of commit c55191e96caa ("arm64: mm: apply r/o permissions of VM areas to its linear alias as well") is that the linear map is created with page granularity, which means that transitioning the early page table from global to non-global mappings when enabling kpti can take a significant amount of time during boot. Given that most CPU implementations do not require kpti, this mainly impacts KASLR builds where kpti is forcefully enabled. However, in these situations we know early on that non-global mappings are required and can avoid the use of global mappings from the beginning. The only gotcha is Cavium erratum #27456, which we must detect based on the MIDR value of the boot CPU. Reviewed-by: Ard Biesheuvel Reported-by: John Garry Signed-off-by: Will Deacon --- arch/arm64/include/asm/mmu.h | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'arch/arm64/include/asm/mmu.h') diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 7689c7aa1d77..ac352accb3d9 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -16,6 +16,8 @@ #ifndef __ASM_MMU_H #define __ASM_MMU_H +#include + #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ #define USER_ASID_BIT 48 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) @@ -44,6 +46,45 @@ static inline bool arm64_kernel_unmapped_at_el0(void) cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0); } +static inline bool arm64_kernel_use_ng_mappings(void) +{ + bool tx1_bug; + + /* What's a kpti? Use global mappings if we don't know. */ + if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) + return false; + + /* + * Note: this function is called before the CPU capabilities have + * been configured, so our early mappings will be global. If we + * later determine that kpti is required, then + * kpti_install_ng_mappings() will make them non-global. + */ + if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE)) + return arm64_kernel_unmapped_at_el0(); + + /* + * KASLR is enabled so we're going to be enabling kpti on non-broken + * CPUs regardless of their susceptibility to Meltdown. Rather + * than force everybody to go through the G -> nG dance later on, + * just put down non-global mappings from the beginning. + */ + if (!IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) { + tx1_bug = false; +#ifndef MODULE + } else if (!static_branch_likely(&arm64_const_caps_ready)) { + extern const struct midr_range cavium_erratum_27456_cpus[]; + + tx1_bug = is_midr_in_range_list(read_cpuid_id(), + cavium_erratum_27456_cpus); +#endif + } else { + tx1_bug = __cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456); + } + + return !tx1_bug && kaslr_offset() > 0; +} + typedef void (*bp_hardening_cb_t)(void); struct bp_hardening_data { -- cgit v1.2.3 From 2f97967503df8e45bc256a348b6f050abd2a38ed Mon Sep 17 00:00:00 2001 From: James Morse Date: Tue, 15 Jan 2019 18:49:17 +0000 Subject: arm64: kpti: Update arm64_kernel_use_ng_mappings() when forced on Since commit b89d82ef01b3 ("arm64: kpti: Avoid rewriting early page tables when KASLR is enabled"), a kernel built with CONFIG_RANDOMIZE_BASE can decide early whether to use non-global mappings by checking the kaslr_offset(). A kernel built without CONFIG_RANDOMIZE_BASE, instead checks the cpufeature static-key. This leaves a gap where CONFIG_RANDOMIZE_BASE was enabled, no kaslr seed was provided, but kpti was forced on using the cmdline option. When the decision is made late, kpti_install_ng_mappings() will re-write the page tables, but arm64_kernel_use_ng_mappings()'s value does not change as it only tests the cpufeature static-key if CONFIG_RANDOMIZE_BASE is disabled. This function influences PROT_DEFAULT via PTE_MAYBE_NG, and causes pgattr_change_is_safe() to catch nG->G transitions when the unchanged PROT_DEFAULT is used as part of PAGE_KERNEL_RO: [ 1.942255] alternatives: patching kernel code [ 1.998288] ------------[ cut here ]------------ [ 2.000693] kernel BUG at arch/arm64/mm/mmu.c:165! [ 2.019215] Internal error: Oops - BUG: 0 [#1] PREEMPT SMP [ 2.020257] Modules linked in: [ 2.020807] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.0.0-rc2 #51 [ 2.021917] Hardware name: linux,dummy-virt (DT) [ 2.022790] pstate: 40000005 (nZcv daif -PAN -UAO) [ 2.023742] pc : __create_pgd_mapping+0x508/0x6d0 [ 2.024671] lr : __create_pgd_mapping+0x500/0x6d0 [ 2.058059] Process swapper/0 (pid: 1, stack limit = 0x(____ptrval____)) [ 2.059369] Call trace: [ 2.059845] __create_pgd_mapping+0x508/0x6d0 [ 2.060684] update_mapping_prot+0x48/0xd0 [ 2.061477] mark_linear_text_alias_ro+0xdc/0xe4 [ 2.070502] smp_cpus_done+0x90/0x98 [ 2.071216] smp_init+0x100/0x114 [ 2.071878] kernel_init_freeable+0xd4/0x220 [ 2.072750] kernel_init+0x10/0x100 [ 2.073455] ret_from_fork+0x10/0x18 [ 2.075414] ---[ end trace 3572f3a7782292de ]--- [ 2.076389] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b If arm64_kernel_unmapped_at_el0() is true, arm64_kernel_use_ng_mappings() should also be true. Signed-off-by: James Morse CC: Ard Biesheuvel CC: John Garry CC: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/mmu.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'arch/arm64/include/asm/mmu.h') diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index ac352accb3d9..3e8063f4f9d3 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -60,8 +60,11 @@ static inline bool arm64_kernel_use_ng_mappings(void) * later determine that kpti is required, then * kpti_install_ng_mappings() will make them non-global. */ + if (arm64_kernel_unmapped_at_el0()) + return true; + if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE)) - return arm64_kernel_unmapped_at_el0(); + return false; /* * KASLR is enabled so we're going to be enabling kpti on non-broken -- cgit v1.2.3