From b89d82ef01b33bc50cbaa8ff05607879b40d0704 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 8 Jan 2019 16:19:01 +0000 Subject: arm64: kpti: Avoid rewriting early page tables when KASLR is enabled A side effect of commit c55191e96caa ("arm64: mm: apply r/o permissions of VM areas to its linear alias as well") is that the linear map is created with page granularity, which means that transitioning the early page table from global to non-global mappings when enabling kpti can take a significant amount of time during boot. Given that most CPU implementations do not require kpti, this mainly impacts KASLR builds where kpti is forcefully enabled. However, in these situations we know early on that non-global mappings are required and can avoid the use of global mappings from the beginning. The only gotcha is Cavium erratum #27456, which we must detect based on the MIDR value of the boot CPU. Reviewed-by: Ard Biesheuvel Reported-by: John Garry Signed-off-by: Will Deacon --- arch/arm64/kernel/cpu_errata.c | 2 +- arch/arm64/kernel/cpufeature.c | 9 +++++++-- arch/arm64/kernel/head.S | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 09ac548c9d44..9950bb0cbd52 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -553,7 +553,7 @@ static const struct midr_range arm64_repeat_tlbi_cpus[] = { #endif #ifdef CONFIG_CAVIUM_ERRATUM_27456 -static const struct midr_range cavium_erratum_27456_cpus[] = { +const struct midr_range cavium_erratum_27456_cpus[] = { /* Cavium ThunderX, T88 pass 1.x - 2.1 */ MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1), /* Cavium ThunderX, T81 pass 1.0 */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 4f272399de89..f6d84e2c92fe 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -983,7 +983,7 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry, /* Useful for KASLR robustness */ if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) - return true; + return kaslr_offset() > 0; /* Don't force KPTI for CPUs that are not vulnerable */ if (is_midr_in_range_list(read_cpuid_id(), kpti_safe_list)) @@ -1003,7 +1003,12 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) static bool kpti_applied = false; int cpu = smp_processor_id(); - if (kpti_applied) + /* + * We don't need to rewrite the page-tables if either we've done + * it already or we have KASLR enabled and therefore have not + * created any global mappings at all. + */ + if (kpti_applied || kaslr_offset() > 0) return; remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index c7213674cb24..15d79a8e5e5e 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -475,6 +475,7 @@ ENDPROC(__primary_switched) ENTRY(kimage_vaddr) .quad _text - TEXT_OFFSET +EXPORT_SYMBOL(kimage_vaddr) /* * If we're fortunate enough to boot at EL2, ensure that the world is -- cgit v1.2.3 From 279667212ab2a4f36c3b0347657ddcc11f9cfa25 Mon Sep 17 00:00:00 2001 From: AKASHI Takahiro Date: Fri, 11 Jan 2019 16:40:21 +0900 Subject: arm64: kexec_file: return successfully even if kaslr-seed doesn't exist In kexec_file_load, kaslr-seed property of the current dtb will be deleted any way before setting a new value if possible. It doesn't matter whether it exists in the current dtb. So "ret" should be reset to 0 here. Fixes: commit 884143f60c89 ("arm64: kexec_file: add kaslr support") Signed-off-by: AKASHI Takahiro Signed-off-by: Will Deacon --- arch/arm64/kernel/machine_kexec_file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index 10e33860e47a..f2c211a6229b 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -87,7 +87,9 @@ static int setup_dtb(struct kimage *image, /* add kaslr-seed */ ret = fdt_delprop(dtb, off, FDT_PROP_KASLR_SEED); - if (ret && (ret != -FDT_ERR_NOTFOUND)) + if (ret == -FDT_ERR_NOTFOUND) + ret = 0; + else if (ret) goto out; if (rng_is_initialized()) { -- cgit v1.2.3 From 1598ecda7b239e9232dda032bfddeed9d89fab6c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 15 Jan 2019 20:47:07 +0100 Subject: arm64: kaslr: ensure randomized quantities are clean to the PoC kaslr_early_init() is called with the kernel mapped at its link time offset, and if it returns with a non-zero offset, the kernel is unmapped and remapped again at the randomized offset. During its execution, kaslr_early_init() also randomizes the base of the module region and of the linear mapping of DRAM, and sets two variables accordingly. However, since these variables are assigned with the caches on, they may get lost during the cache maintenance that occurs when unmapping and remapping the kernel, so ensure that these values are cleaned to the PoC. Acked-by: Catalin Marinas Fixes: f80fb3a3d508 ("arm64: add support for kernel ASLR") Cc: # v4.6+ Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/kernel/kaslr.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index f0e6ab8abe9c..ba6b41790fcd 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -43,7 +44,7 @@ static __init u64 get_kaslr_seed(void *fdt) return ret; } -static __init const u8 *get_cmdline(void *fdt) +static __init const u8 *kaslr_get_cmdline(void *fdt) { static __initconst const u8 default_cmdline[] = CONFIG_CMDLINE; @@ -109,7 +110,7 @@ u64 __init kaslr_early_init(u64 dt_phys) * Check if 'nokaslr' appears on the command line, and * return 0 if that is the case. */ - cmdline = get_cmdline(fdt); + cmdline = kaslr_get_cmdline(fdt); str = strstr(cmdline, "nokaslr"); if (str == cmdline || (str > cmdline && *(str - 1) == ' ')) return 0; @@ -169,5 +170,8 @@ u64 __init kaslr_early_init(u64 dt_phys) module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21; module_alloc_base &= PAGE_MASK; + __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base)); + __flush_dcache_area(&memstart_offset_seed, sizeof(memstart_offset_seed)); + return offset; } -- cgit v1.2.3 From 8ea235932314311f15ea6cf65c1393ed7e31af70 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 27 Jan 2019 09:29:42 +0100 Subject: arm64: kaslr: ensure randomized quantities are clean also when kaslr is off Commit 1598ecda7b23 ("arm64: kaslr: ensure randomized quantities are clean to the PoC") added cache maintenance to ensure that global variables set by the kaslr init routine are not wiped clean due to cache invalidation occurring during the second round of page table creation. However, if kaslr_early_init() exits early with no randomization being applied (either due to the lack of a seed, or because the user has disabled kaslr explicitly), no cache maintenance is performed, leading to the same issue we attempted to fix earlier, as far as the module_alloc_base variable is concerned. Note that module_alloc_base cannot be initialized statically, because that would cause it to be subject to a R_AARCH64_RELATIVE relocation, causing it to be overwritten by the second round of KASLR relocation processing. Fixes: f80fb3a3d508 ("arm64: add support for kernel ASLR") Cc: # v4.6+ Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/kernel/kaslr.c | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index ba6b41790fcd..b09b6f75f759 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -88,6 +88,7 @@ u64 __init kaslr_early_init(u64 dt_phys) * we end up running with module randomization disabled. */ module_alloc_base = (u64)_etext - MODULES_VSIZE; + __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base)); /* * Try to map the FDT early. If this fails, we simply bail, -- cgit v1.2.3 From f2b3d8566d81deaca31f4e3163def0bea7746e11 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 24 Jan 2019 16:32:55 +0000 Subject: arm64: kprobe: Always blacklist the KVM world-switch code On systems with VHE the kernel and KVM's world-switch code run at the same exception level. Code that is only used on a VHE system does not need to be annotated as __hyp_text as it can reside anywhere in the kernel text. __hyp_text was also used to prevent kprobes from patching breakpoint instructions into this region, as this code runs at a different exception level. While this is no longer true with VHE, KVM still switches VBAR_EL1, meaning a kprobe's breakpoint executed in the world-switch code will cause a hyp-panic. Move the __hyp_text check in the kprobes blacklist so it applies on VHE systems too, to cover the common code and guest enter/exit assembly. Fixes: 888b3c8720e0 ("arm64: Treat all entry code as non-kprobe-able") Reviewed-by: Christoffer Dall Signed-off-by: James Morse Acked-by: Masami Hiramatsu Signed-off-by: Will Deacon --- arch/arm64/kernel/probes/kprobes.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 2a5b338b2542..f17afb99890c 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -478,13 +478,13 @@ bool arch_within_kprobe_blacklist(unsigned long addr) addr < (unsigned long)__entry_text_end) || (addr >= (unsigned long)__idmap_text_start && addr < (unsigned long)__idmap_text_end) || + (addr >= (unsigned long)__hyp_text_start && + addr < (unsigned long)__hyp_text_end) || !!search_exception_tables(addr)) return true; if (!is_kernel_in_hyp_mode()) { - if ((addr >= (unsigned long)__hyp_text_start && - addr < (unsigned long)__hyp_text_end) || - (addr >= (unsigned long)__hyp_idmap_text_start && + if ((addr >= (unsigned long)__hyp_idmap_text_start && addr < (unsigned long)__hyp_idmap_text_end)) return true; } -- cgit v1.2.3 From 8fac5cbdfe0f01254d9d265c6aa1a95f94f58595 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 24 Jan 2019 16:32:56 +0000 Subject: arm64: hyp-stub: Forbid kprobing of the hyp-stub The hyp-stub is loaded by the kernel's early startup code at EL2 during boot, before KVM takes ownership later. The hyp-stub's text is part of the regular kernel text, meaning it can be kprobed. A breakpoint in the hyp-stub causes the CPU to spin in el2_sync_invalid. Add it to the __hyp_text. Signed-off-by: James Morse Cc: stable@vger.kernel.org Signed-off-by: Will Deacon --- arch/arm64/kernel/hyp-stub.S | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index e1261fbaa374..17f325ba831e 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -28,6 +28,8 @@ #include .text + .pushsection .hyp.text, "ax" + .align 11 ENTRY(__hyp_stub_vectors) -- cgit v1.2.3 From f7daa9c8fd191724b9ab9580a7be55cd1a67d799 Mon Sep 17 00:00:00 2001 From: James Morse Date: Thu, 24 Jan 2019 16:32:57 +0000 Subject: arm64: hibernate: Clean the __hyp_text to PoC after resume During resume hibernate restores all physical memory. Any memory that is accessed with the MMU disabled needs to be cleaned to the PoC. KVMs __hyp_text was previously ommitted as it runs with the MMU enabled, but now that the hyp-stub is located in this section, we must clean __hyp_text too. This ensures secondary CPUs that come online after hibernate has finished resuming, and load KVM via the freshly written hyp-stub see the correct instructions. Signed-off-by: James Morse Cc: stable@vger.kernel.org Signed-off-by: Will Deacon --- arch/arm64/kernel/hibernate.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 29cdc99688f3..9859e1178e6b 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -299,8 +299,10 @@ int swsusp_arch_suspend(void) dcache_clean_range(__idmap_text_start, __idmap_text_end); /* Clean kvm setup code to PoC? */ - if (el2_reset_needed()) + if (el2_reset_needed()) { dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); + dcache_clean_range(__hyp_text_start, __hyp_text_end); + } /* make the crash dump kernel image protected again */ crash_post_resume(); -- cgit v1.2.3 From ea5736805190e912903c27c9f17c7a4341a405e9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 1 Feb 2019 16:26:08 +0000 Subject: arm64: kexec_file: handle empty command-line Calling strlen() on cmdline == NULL produces a kernel oops. Since having a NULL cmdline is valid, handle this case explicitly. Fixes: 52b2a8af7436 ("arm64: kexec_file: load initrd and device-tree") Signed-off-by: Jean-Philippe Brucker Signed-off-by: Will Deacon --- arch/arm64/kernel/machine_kexec_file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index f2c211a6229b..58871333737a 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -120,10 +120,12 @@ static int create_dtb(struct kimage *image, { void *buf; size_t buf_size; + size_t cmdline_len; int ret; + cmdline_len = cmdline ? strlen(cmdline) : 0; buf_size = fdt_totalsize(initial_boot_params) - + strlen(cmdline) + DTB_EXTRA_SPACE; + + cmdline_len + DTB_EXTRA_SPACE; for (;;) { buf = vmalloc(buf_size); -- cgit v1.2.3