diff options
| author | Ard Biesheuvel <ardb@kernel.org> | 2021-11-24 14:08:11 +0100 |
|---|---|---|
| committer | Ard Biesheuvel <ardb@kernel.org> | 2021-12-06 12:49:17 +0100 |
| commit | 9c46929e7989efacc1dd0a1dd662a839897ea2b6 (patch) | |
| tree | e55a231cce8f02eea3be6e7b19627c58bccdee52 /arch/arm/include/asm/current.h | |
| parent | c2755910373bb5dfb9aa68ba2924036686815c9e (diff) | |
| download | linux-9c46929e7989efacc1dd0a1dd662a839897ea2b6.tar.bz2 | |
ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems
On UP systems, only a single task can be 'current' at the same time,
which means we can use a global variable to track it. This means we can
also enable THREAD_INFO_IN_TASK for those systems, as in that case,
thread_info is accessed via current rather than the other way around,
removing the need to store thread_info at the base of the task stack.
This, in turn, permits us to enable IRQ stacks and vmap'ed stacks on UP
systems as well.
To partially mitigate the performance overhead of this arrangement, use
a ADD/ADD/LDR sequence with the appropriate PC-relative group
relocations to load the value of current when needed. This means that
accessing current will still only require a single load as before,
avoiding the need for a literal to carry the address of the global
variable in each function. However, accessing thread_info will now
require this load as well.
Acked-by: Linus Walleij <linus.walleij@linaro.org>
Acked-by: Nicolas Pitre <nico@fluxnic.net>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Tested-by: Marc Zyngier <maz@kernel.org>
Tested-by: Vladimir Murzin <vladimir.murzin@arm.com> # ARMv7M
Diffstat (limited to 'arch/arm/include/asm/current.h')
| -rw-r--r-- | arch/arm/include/asm/current.h | 37 |
1 files changed, 22 insertions, 15 deletions
diff --git a/arch/arm/include/asm/current.h b/arch/arm/include/asm/current.h index 6bf0aad672c3..69ecf4c6c725 100644 --- a/arch/arm/include/asm/current.h +++ b/arch/arm/include/asm/current.h @@ -8,25 +8,18 @@ #define _ASM_ARM_CURRENT_H #ifndef __ASSEMBLY__ +#include <asm/insn.h> struct task_struct; -static inline void set_current(struct task_struct *cur) -{ - if (!IS_ENABLED(CONFIG_CURRENT_POINTER_IN_TPIDRURO)) - return; - - /* Set TPIDRURO */ - asm("mcr p15, 0, %0, c13, c0, 3" :: "r"(cur) : "memory"); -} +extern struct task_struct *__current; -#ifdef CONFIG_CURRENT_POINTER_IN_TPIDRURO - -static inline struct task_struct *get_current(void) +static inline __attribute_const__ struct task_struct *get_current(void) { struct task_struct *cur; #if __has_builtin(__builtin_thread_pointer) && \ + defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) && \ !(defined(CONFIG_THUMB2_KERNEL) && \ defined(CONFIG_CC_IS_CLANG) && CONFIG_CLANG_VERSION < 130001) /* @@ -39,16 +32,30 @@ static inline struct task_struct *get_current(void) * https://github.com/ClangBuiltLinux/linux/issues/1485 */ cur = __builtin_thread_pointer(); +#elif defined(CONFIG_CURRENT_POINTER_IN_TPIDRURO) || defined(CONFIG_SMP) + asm("0: mrc p15, 0, %0, c13, c0, 3 \n\t" +#ifdef CONFIG_CPU_V6 + "1: \n\t" + " .subsection 1 \n\t" + "2: " LOAD_SYM_ARMV6(%0, __current) " \n\t" + " b 1b \n\t" + " .previous \n\t" + " .pushsection \".alt.smp.init\", \"a\" \n\t" + " .long 0b - . \n\t" + " b . + (2b - 0b) \n\t" + " .popsection \n\t" +#endif + : "=r"(cur)); +#elif __LINUX_ARM_ARCH__>=7 || \ + (defined(MODULE) && defined(CONFIG_ARM_MODULE_PLTS)) + cur = __current; #else - asm("mrc p15, 0, %0, c13, c0, 3" : "=r"(cur)); + asm(LOAD_SYM_ARMV6(%0, __current) : "=r"(cur)); #endif return cur; } #define current get_current() -#else -#include <asm-generic/current.h> -#endif /* CONFIG_CURRENT_POINTER_IN_TPIDRURO */ #endif /* __ASSEMBLY__ */ |