From 3c5f7e7b4a0346de670b08f595bd15e7eec91f97 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 31 May 2011 15:38:43 +0100 Subject: ARM: Use TTBR1 instead of reserved context ID On ARMv7 CPUs that cache first level page table entries (like the Cortex-A15), using a reserved ASID while changing the TTBR or flushing the TLB is unsafe. This is because the CPU may cache the first level entry as the result of a speculative memory access while the reserved ASID is assigned. After the process owning the page tables dies, the memory will be reallocated and may be written with junk values which can be interpreted as global, valid PTEs by the processor. This will result in the TLB being populated with bogus global entries. This patch avoids the use of a reserved context ID in the v7 switch_mm and ASID rollover code by temporarily using the swapper_pg_dir pointed at by TTBR1, which contains only global entries that are not tagged with ASIDs. Reviewed-by: Frank Rowand Tested-by: Marc Zyngier Signed-off-by: Will Deacon [catalin.marinas@arm.com: add LPAE support] Signed-off-by: Catalin Marinas --- arch/arm/mm/context.c | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) (limited to 'arch/arm/mm/context.c') diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index ee9bb363d606..aaa291fc072e 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -23,25 +23,37 @@ DEFINE_PER_CPU(struct mm_struct *, current_mm); #endif #ifdef CONFIG_ARM_LPAE -#define cpu_set_asid(asid) { \ - unsigned long ttbl, ttbh; \ - asm volatile( \ - " mrrc p15, 0, %0, %1, c2 @ read TTBR0\n" \ - " mov %1, %2, lsl #(48 - 32) @ set ASID\n" \ - " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" \ - : "=&r" (ttbl), "=&r" (ttbh) \ - : "r" (asid & ~ASID_MASK)); \ +static void cpu_set_reserved_ttbr0(void) +{ + unsigned long ttbl = __pa(swapper_pg_dir); + unsigned long ttbh = 0; + + /* + * Set TTBR0 to swapper_pg_dir which contains only global entries. The + * ASID is set to 0. + */ + asm volatile( + " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" + : + : "r" (ttbl), "r" (ttbh)); + isb(); } #else -#define cpu_set_asid(asid) \ - asm(" mcr p15, 0, %0, c13, c0, 1\n" : : "r" (asid)) +static void cpu_set_reserved_ttbr0(void) +{ + u32 ttb; + /* Copy TTBR1 into TTBR0 */ + asm volatile( + " mrc p15, 0, %0, c2, c0, 1 @ read TTBR1\n" + " mcr p15, 0, %0, c2, c0, 0 @ set TTBR0\n" + : "=r" (ttb)); + isb(); +} #endif /* * We fork()ed a process, and we need a new context for the child - * to run in. We reserve version 0 for initial tasks so we will - * always allocate an ASID. The ASID 0 is reserved for the TTBR - * register changing sequence. + * to run in. */ void __init_new_context(struct task_struct *tsk, struct mm_struct *mm) { @@ -51,9 +63,7 @@ void __init_new_context(struct task_struct *tsk, struct mm_struct *mm) static void flush_context(void) { - /* set the reserved ASID before flushing the TLB */ - cpu_set_asid(0); - isb(); + cpu_set_reserved_ttbr0(); local_flush_tlb_all(); if (icache_is_vivt_asid_tagged()) { __flush_icache_all(); @@ -114,8 +124,7 @@ static void reset_context(void *info) set_mm_context(mm, asid); /* set the new ASID */ - cpu_set_asid(mm->context.id); - isb(); + cpu_switch_mm(mm->pgd, mm); } #else -- cgit v1.2.3 From 7fec1b57b8a925d83c194f995f83d9f8442fd48e Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 28 Nov 2011 13:53:28 +0000 Subject: ARM: Remove __ARCH_WANT_INTERRUPTS_ON_CTXSW on ASID-capable CPUs Since the ASIDs must be unique to an mm across all the CPUs in a system, the __new_context() function needs to broadcast a context reset event to all the CPUs during ASID allocation if a roll-over occurred. Such IPIs cannot be issued with interrupts disabled and ARM had to define __ARCH_WANT_INTERRUPTS_ON_CTXSW. This patch changes the check_context() function to check_and_switch_context() called from switch_mm(). In case of ASID-capable CPUs (ARMv6 onwards), if a new ASID is needed and the interrupts are disabled, it defers the __new_context() and cpu_switch_mm() calls to the post-lock switch hook where the interrupts are enabled. Setting the reserved TTBR0 was also moved to check_and_switch_context() from cpu_v7_switch_mm(). Reviewed-by: Will Deacon Tested-by: Will Deacon Reviewed-by: Frank Rowand Tested-by: Marc Zyngier Signed-off-by: Catalin Marinas --- arch/arm/include/asm/mmu.h | 2 ++ arch/arm/include/asm/mmu_context.h | 72 +++++++++++++++++++++++++++++--------- arch/arm/include/asm/thread_info.h | 1 + arch/arm/mm/context.c | 4 +-- arch/arm/mm/proc-v7-2level.S | 3 -- 5 files changed, 61 insertions(+), 21 deletions(-) (limited to 'arch/arm/mm/context.c') diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index b8e580a297e4..20b43d6f23b3 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -39,6 +39,8 @@ typedef struct { * so enable interrupts over the context switch to avoid high * latency. */ +#ifndef CONFIG_CPU_HAS_ASID #define __ARCH_WANT_INTERRUPTS_ON_CTXSW +#endif #endif diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index a0b3cac0547c..94e265cb5146 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -49,39 +49,80 @@ DECLARE_PER_CPU(struct mm_struct *, current_mm); void __init_new_context(struct task_struct *tsk, struct mm_struct *mm); void __new_context(struct mm_struct *mm); +void cpu_set_reserved_ttbr0(void); -static inline void check_context(struct mm_struct *mm) +static inline void switch_new_context(struct mm_struct *mm) { - /* - * This code is executed with interrupts enabled. Therefore, - * mm->context.id cannot be updated to the latest ASID version - * on a different CPU (and condition below not triggered) - * without first getting an IPI to reset the context. The - * alternative is to take a read_lock on mm->context.id_lock - * (after changing its type to rwlock_t). - */ - if (unlikely((mm->context.id ^ cpu_last_asid) >> ASID_BITS)) - __new_context(mm); + unsigned long flags; + __new_context(mm); + + local_irq_save(flags); + cpu_switch_mm(mm->pgd, mm); + local_irq_restore(flags); +} + +static inline void check_and_switch_context(struct mm_struct *mm, + struct task_struct *tsk) +{ if (unlikely(mm->context.kvm_seq != init_mm.context.kvm_seq)) __check_kvm_seq(mm); + + /* + * Required during context switch to avoid speculative page table + * walking with the wrong TTBR. + */ + cpu_set_reserved_ttbr0(); + + if (!((mm->context.id ^ cpu_last_asid) >> ASID_BITS)) + /* + * The ASID is from the current generation, just switch to the + * new pgd. This condition is only true for calls from + * context_switch() and interrupts are already disabled. + */ + cpu_switch_mm(mm->pgd, mm); + else if (irqs_disabled()) + /* + * Defer the new ASID allocation until after the context + * switch critical region since __new_context() cannot be + * called with interrupts disabled (it sends IPIs). + */ + set_ti_thread_flag(task_thread_info(tsk), TIF_SWITCH_MM); + else + /* + * That is a direct call to switch_mm() or activate_mm() with + * interrupts enabled and a new context. + */ + switch_new_context(mm); } #define init_new_context(tsk,mm) (__init_new_context(tsk,mm),0) -#else +#define finish_arch_post_lock_switch \ + finish_arch_post_lock_switch +static inline void finish_arch_post_lock_switch(void) +{ + if (test_and_clear_thread_flag(TIF_SWITCH_MM)) + switch_new_context(current->mm); +} -static inline void check_context(struct mm_struct *mm) +#else /* !CONFIG_CPU_HAS_ASID */ + +static inline void check_and_switch_context(struct mm_struct *mm, + struct task_struct *tsk) { #ifdef CONFIG_MMU if (unlikely(mm->context.kvm_seq != init_mm.context.kvm_seq)) __check_kvm_seq(mm); + cpu_switch_mm(mm->pgd, mm); #endif } #define init_new_context(tsk,mm) 0 -#endif +#define finish_arch_post_lock_switch() do { } while (0) + +#endif /* CONFIG_CPU_HAS_ASID */ #define destroy_context(mm) do { } while(0) @@ -123,8 +164,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, struct mm_struct **crt_mm = &per_cpu(current_mm, cpu); *crt_mm = next; #endif - check_context(next); - cpu_switch_mm(next->pgd, next); + check_and_switch_context(next, tsk); if (cache_is_vivt()) cpumask_clear_cpu(cpu, mm_cpumask(prev)); } diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index d4c24d412a8d..9e13e33ec746 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -146,6 +146,7 @@ extern void vfp_flush_hwstate(struct thread_info *); #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ #define TIF_RESTORE_SIGMASK 20 #define TIF_SECCOMP 21 +#define TIF_SWITCH_MM 22 /* deferred switch_mm */ #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index aaa291fc072e..06a2e7ce23c3 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -23,7 +23,7 @@ DEFINE_PER_CPU(struct mm_struct *, current_mm); #endif #ifdef CONFIG_ARM_LPAE -static void cpu_set_reserved_ttbr0(void) +void cpu_set_reserved_ttbr0(void) { unsigned long ttbl = __pa(swapper_pg_dir); unsigned long ttbh = 0; @@ -39,7 +39,7 @@ static void cpu_set_reserved_ttbr0(void) isb(); } #else -static void cpu_set_reserved_ttbr0(void) +void cpu_set_reserved_ttbr0(void) { u32 ttb; /* Copy TTBR1 into TTBR0 */ diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S index 72270482a922..42ac069c8012 100644 --- a/arch/arm/mm/proc-v7-2level.S +++ b/arch/arm/mm/proc-v7-2level.S @@ -46,9 +46,6 @@ ENTRY(cpu_v7_switch_mm) #ifdef CONFIG_ARM_ERRATA_430973 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB #endif - mrc p15, 0, r2, c2, c0, 1 @ load TTB 1 - mcr p15, 0, r2, c2, c0, 0 @ into TTB 0 - isb #ifdef CONFIG_ARM_ERRATA_754322 dsb #endif -- cgit v1.2.3 From e323969ccda2d69f02e047c08b03faa09215c72a Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 28 Nov 2011 15:59:10 +0000 Subject: ARM: Remove current_mm per-cpu variable The current_mm variable was used to store the new mm between the switch_mm() and switch_to() calls where an IPI to reset the context could have set the wrong mm. Since the interrupts are disabled during context switch, there is no need for this variable, current->active_mm already points to the current mm when interrupts are re-enabled. Reviewed-by: Will Deacon Tested-by: Will Deacon Reviewed-by: Frank Rowand Tested-by: Marc Zyngier Signed-off-by: Catalin Marinas --- arch/arm/include/asm/mmu_context.h | 7 ------- arch/arm/mm/context.c | 12 +----------- 2 files changed, 1 insertion(+), 18 deletions(-) (limited to 'arch/arm/mm/context.c') diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index 94e265cb5146..8da4b9c042fe 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -43,9 +43,6 @@ void __check_kvm_seq(struct mm_struct *mm); #define ASID_FIRST_VERSION (1 << ASID_BITS) extern unsigned int cpu_last_asid; -#ifdef CONFIG_SMP -DECLARE_PER_CPU(struct mm_struct *, current_mm); -#endif void __init_new_context(struct task_struct *tsk, struct mm_struct *mm); void __new_context(struct mm_struct *mm); @@ -160,10 +157,6 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, __flush_icache_all(); #endif if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next) { -#ifdef CONFIG_SMP - struct mm_struct **crt_mm = &per_cpu(current_mm, cpu); - *crt_mm = next; -#endif check_and_switch_context(next, tsk); if (cache_is_vivt()) cpumask_clear_cpu(cpu, mm_cpumask(prev)); diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index 06a2e7ce23c3..806cc4f63516 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -18,9 +18,6 @@ static DEFINE_RAW_SPINLOCK(cpu_asid_lock); unsigned int cpu_last_asid = ASID_FIRST_VERSION; -#ifdef CONFIG_SMP -DEFINE_PER_CPU(struct mm_struct *, current_mm); -#endif #ifdef CONFIG_ARM_LPAE void cpu_set_reserved_ttbr0(void) @@ -108,14 +105,7 @@ static void reset_context(void *info) { unsigned int asid; unsigned int cpu = smp_processor_id(); - struct mm_struct *mm = per_cpu(current_mm, cpu); - - /* - * Check if a current_mm was set on this CPU as it might still - * be in the early booting stages and using the reserved ASID. - */ - if (!mm) - return; + struct mm_struct *mm = current->active_mm; smp_rmb(); asid = cpu_last_asid + cpu + 1; -- cgit v1.2.3