From f419e6f63c5afea00d7c17ebf54f2d265f5c4d7e Mon Sep 17 00:00:00 2001 From: Zhigang Lu Date: Wed, 30 Sep 2015 09:53:47 +0800 Subject: tile: define a macro ktext_writable_addr to get writable kernel text address It is used by kgdb, ftrace, kprobe and jump label, so we factor this out into a helper routine. Reviewed-by: Chris Metcalf Signed-off-by: Zhigang Lu Signed-off-by: Chris Metcalf --- arch/tile/include/asm/page.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'arch/tile/include') diff --git a/arch/tile/include/asm/page.h b/arch/tile/include/asm/page.h index a213a8d84a95..5cee2cbff2b1 100644 --- a/arch/tile/include/asm/page.h +++ b/arch/tile/include/asm/page.h @@ -319,6 +319,16 @@ static inline int pfn_valid(unsigned long pfn) #define virt_to_page(kaddr) pfn_to_page(kaddr_to_pfn((void *)(kaddr))) #define page_to_virt(page) pfn_to_kaddr(page_to_pfn(page)) +/* + * The kernel text is mapped at MEM_SV_START as read-only. To allow + * modifying kernel text, it is also mapped at PAGE_OFFSET as read-write. + * This macro converts a kernel address to its writable kernel text mapping, + * which is used to modify the text code on a running kernel by kgdb, + * ftrace, kprobe, jump label, etc. + */ +#define ktext_writable_addr(kaddr) \ + ((unsigned long)(kaddr) - MEM_SV_START + PAGE_OFFSET) + struct mm_struct; extern pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr); extern pte_t *virt_to_kpte(unsigned long kaddr); -- cgit v1.2.3 From 65a792e84f25d1436698f999224b2cf5d7594546 Mon Sep 17 00:00:00 2001 From: Zhigang Lu Date: Wed, 30 Sep 2015 10:11:45 +0800 Subject: tile/jump_label: add jump label support for TILE-Gx Add the arch-specific code to support jump label for TILE-Gx. This code shares NOP instruction with ftrace, so we move it to a common header file. Reviewed-by: Chris Metcalf Signed-off-by: Zhigang Lu Signed-off-by: Chris Metcalf --- arch/tile/Kconfig | 1 + arch/tile/include/asm/insn.h | 59 +++++++++++++++++++++++++++++++++++ arch/tile/include/asm/jump_label.h | 58 ++++++++++++++++++++++++++++++++++ arch/tile/kernel/Makefile | 1 + arch/tile/kernel/ftrace.c | 11 +------ arch/tile/kernel/jump_label.c | 64 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 184 insertions(+), 10 deletions(-) create mode 100644 arch/tile/include/asm/insn.h create mode 100644 arch/tile/include/asm/jump_label.h create mode 100644 arch/tile/kernel/jump_label.c (limited to 'arch/tile/include') diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig index 106c21bd7f44..49be476f9feb 100644 --- a/arch/tile/Kconfig +++ b/arch/tile/Kconfig @@ -143,6 +143,7 @@ config TILEGX select HAVE_KRETPROBES select HAVE_ARCH_KGDB select ARCH_SUPPORTS_ATOMIC_RMW + select HAVE_ARCH_JUMP_LABEL config TILEPRO def_bool !TILEGX diff --git a/arch/tile/include/asm/insn.h b/arch/tile/include/asm/insn.h new file mode 100644 index 000000000000..f78ba5c16722 --- /dev/null +++ b/arch/tile/include/asm/insn.h @@ -0,0 +1,59 @@ +/* + * Copyright 2015 Tilera Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for + * more details. + */ +#ifndef __ASM_TILE_INSN_H +#define __ASM_TILE_INSN_H + +#include + +static inline tilegx_bundle_bits NOP(void) +{ + return create_UnaryOpcodeExtension_X0(FNOP_UNARY_OPCODE_X0) | + create_RRROpcodeExtension_X0(UNARY_RRR_0_OPCODE_X0) | + create_Opcode_X0(RRR_0_OPCODE_X0) | + create_UnaryOpcodeExtension_X1(NOP_UNARY_OPCODE_X1) | + create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | + create_Opcode_X1(RRR_0_OPCODE_X1); +} + +static inline tilegx_bundle_bits tilegx_gen_branch(unsigned long pc, + unsigned long addr, + bool link) +{ + tilegx_bundle_bits opcode_x0, opcode_x1; + long pcrel_by_instr = (addr - pc) >> TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES; + + if (link) { + /* opcode: jal addr */ + opcode_x1 = + create_Opcode_X1(JUMP_OPCODE_X1) | + create_JumpOpcodeExtension_X1(JAL_JUMP_OPCODE_X1) | + create_JumpOff_X1(pcrel_by_instr); + } else { + /* opcode: j addr */ + opcode_x1 = + create_Opcode_X1(JUMP_OPCODE_X1) | + create_JumpOpcodeExtension_X1(J_JUMP_OPCODE_X1) | + create_JumpOff_X1(pcrel_by_instr); + } + + /* opcode: fnop */ + opcode_x0 = + create_UnaryOpcodeExtension_X0(FNOP_UNARY_OPCODE_X0) | + create_RRROpcodeExtension_X0(UNARY_RRR_0_OPCODE_X0) | + create_Opcode_X0(RRR_0_OPCODE_X0); + + return opcode_x1 | opcode_x0; +} + +#endif /* __ASM_TILE_INSN_H */ diff --git a/arch/tile/include/asm/jump_label.h b/arch/tile/include/asm/jump_label.h new file mode 100644 index 000000000000..cde7573f397b --- /dev/null +++ b/arch/tile/include/asm/jump_label.h @@ -0,0 +1,58 @@ +/* + * Copyright 2015 Tilera Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for + * more details. + */ + +#ifndef _ASM_TILE_JUMP_LABEL_H +#define _ASM_TILE_JUMP_LABEL_H + +#include + +#define JUMP_LABEL_NOP_SIZE TILE_BUNDLE_SIZE_IN_BYTES + +static __always_inline bool arch_static_branch(struct static_key *key, + bool branch) +{ + asm_volatile_goto("1:\n\t" + "nop" "\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".quad 1b, %l[l_yes], %0 + %1 \n\t" + ".popsection\n\t" + : : "i" (key), "i" (branch) : : l_yes); + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, + bool branch) +{ + asm_volatile_goto("1:\n\t" + "j %l[l_yes]" "\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".quad 1b, %l[l_yes], %0 + %1 \n\t" + ".popsection\n\t" + : : "i" (key), "i" (branch) : : l_yes); + return false; +l_yes: + return true; +} + +typedef u64 jump_label_t; + +struct jump_entry { + jump_label_t code; + jump_label_t target; + jump_label_t key; +}; + +#endif /* _ASM_TILE_JUMP_LABEL_H */ diff --git a/arch/tile/kernel/Makefile b/arch/tile/kernel/Makefile index 21f77bf68c69..09936d0bcb42 100644 --- a/arch/tile/kernel/Makefile +++ b/arch/tile/kernel/Makefile @@ -32,5 +32,6 @@ obj-$(CONFIG_TILE_HVGLUE_TRACE) += hvglue_trace.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o mcount_64.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_KGDB) += kgdb.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-y += vdso/ diff --git a/arch/tile/kernel/ftrace.c b/arch/tile/kernel/ftrace.c index 4180ccdf9cd0..4a572088b270 100644 --- a/arch/tile/kernel/ftrace.c +++ b/arch/tile/kernel/ftrace.c @@ -20,21 +20,12 @@ #include #include #include +#include #include #ifdef CONFIG_DYNAMIC_FTRACE -static inline tilegx_bundle_bits NOP(void) -{ - return create_UnaryOpcodeExtension_X0(FNOP_UNARY_OPCODE_X0) | - create_RRROpcodeExtension_X0(UNARY_RRR_0_OPCODE_X0) | - create_Opcode_X0(RRR_0_OPCODE_X0) | - create_UnaryOpcodeExtension_X1(NOP_UNARY_OPCODE_X1) | - create_RRROpcodeExtension_X1(UNARY_RRR_0_OPCODE_X1) | - create_Opcode_X1(RRR_0_OPCODE_X1); -} - static int machine_stopped __read_mostly; int ftrace_arch_code_modify_prepare(void) diff --git a/arch/tile/kernel/jump_label.c b/arch/tile/kernel/jump_label.c new file mode 100644 index 000000000000..07802d586988 --- /dev/null +++ b/arch/tile/kernel/jump_label.c @@ -0,0 +1,64 @@ +/* + * Copyright 2015 Tilera Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for + * more details. + * + * jump label TILE-Gx support + */ + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef HAVE_JUMP_LABEL + +static void __jump_label_transform(struct jump_entry *e, + enum jump_label_type type) +{ + tilegx_bundle_bits opcode; + /* Operate on writable kernel text mapping. */ + unsigned long pc_wr = ktext_writable_addr(e->code); + + if (type == JUMP_LABEL_JMP) + opcode = tilegx_gen_branch(e->code, e->target, false); + else + opcode = NOP(); + + *(tilegx_bundle_bits *)pc_wr = opcode; + /* Make sure that above mem writes were issued towards the memory. */ + smp_wmb(); +} + +void arch_jump_label_transform(struct jump_entry *e, + enum jump_label_type type) +{ + get_online_cpus(); + mutex_lock(&text_mutex); + + __jump_label_transform(e, type); + flush_icache_range(e->code, e->code + sizeof(tilegx_bundle_bits)); + + mutex_unlock(&text_mutex); + put_online_cpus(); +} + +__init_or_module void arch_jump_label_transform_static(struct jump_entry *e, + enum jump_label_type type) +{ + __jump_label_transform(e, type); +} + +#endif /* HAVE_JUMP_LABEL */ -- cgit v1.2.3 From 583b24a210ada7e88fc12aaf50024975ec882816 Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Tue, 22 Sep 2015 14:49:41 -0400 Subject: arch/tile: adopt prepare_exit_to_usermode() model from x86 This change is a prerequisite change for TASK_ISOLATION but also stands on its own for readability and maintainability. The existing tile do_work_pending() was called in a loop from assembly on the slow path; this change moves the loop into C code as well. For the x86 version see commit c5c46f59e4e7 ("x86/entry: Add new, comprehensible entry and exit handlers written in C"). This change exposes a pre-existing bug on the older tilepro platform; the singlestep processing is done last, but on tilepro (unlike tilegx) we enable interrupts while doing that processing, so we could in theory miss a signal or other asynchronous event. A future change could fix this by breaking the singlestep work into a "prepare" step done in the main loop, and a "trigger" step done after exiting the loop. Since this change is intended as purely a restructuring change, we call out the bug explicitly now, but don't yet fix it. Signed-off-by: Chris Metcalf --- arch/tile/include/asm/processor.h | 2 +- arch/tile/include/asm/thread_info.h | 8 +++- arch/tile/kernel/intvec_32.S | 46 +++++++-------------- arch/tile/kernel/intvec_64.S | 49 +++++++---------------- arch/tile/kernel/process.c | 79 +++++++++++++++++++------------------ 5 files changed, 77 insertions(+), 107 deletions(-) (limited to 'arch/tile/include') diff --git a/arch/tile/include/asm/processor.h b/arch/tile/include/asm/processor.h index 139dfdee0134..0684e88aacd8 100644 --- a/arch/tile/include/asm/processor.h +++ b/arch/tile/include/asm/processor.h @@ -212,7 +212,7 @@ static inline void release_thread(struct task_struct *dead_task) /* Nothing for now */ } -extern int do_work_pending(struct pt_regs *regs, u32 flags); +extern void prepare_exit_to_usermode(struct pt_regs *regs, u32 flags); /* diff --git a/arch/tile/include/asm/thread_info.h b/arch/tile/include/asm/thread_info.h index dc1fb28d9636..4b7cef9e94e0 100644 --- a/arch/tile/include/asm/thread_info.h +++ b/arch/tile/include/asm/thread_info.h @@ -140,10 +140,14 @@ extern void _cpu_idle(void); #define _TIF_POLLING_NRFLAG (1<flags. It is - * entered with interrupts disabled so we don't miss an event - * that modified the thread_info flags. If any flag is set, we - * handle it and return, and the calling assembly code will - * re-disable interrupts, reload the thread flags, and call back - * if more flags need to be handled. - * - * We return whether we need to check the thread_info flags again - * or not. Note that we don't clear TIF_SINGLESTEP here, so it's - * important that it be tested last, and then claim that we don't - * need to recheck the flags. + * TIF_ALLWORK_MASK flags are set in thread_info->flags. It is + * entered with interrupts disabled so we don't miss an event that + * modified the thread_info flags. We loop until all the tested flags + * are clear. Note that the function is called on certain conditions + * that are not listed in the loop condition here (e.g. SINGLESTEP) + * which guarantees we will do those things once, and redo them if any + * of the other work items is re-done, but won't continue looping if + * all the other work is done. */ -int do_work_pending(struct pt_regs *regs, u32 thread_info_flags) +void prepare_exit_to_usermode(struct pt_regs *regs, u32 thread_info_flags) { - /* If we enter in kernel mode, do nothing and exit the caller loop. */ - if (!user_mode(regs)) - return 0; + if (WARN_ON(!user_mode(regs))) + return; - user_exit(); + do { + local_irq_enable(); - /* Enable interrupts; they are disabled again on return to caller. */ - local_irq_enable(); + if (thread_info_flags & _TIF_NEED_RESCHED) + schedule(); - if (thread_info_flags & _TIF_NEED_RESCHED) { - schedule(); - return 1; - } #if CHIP_HAS_TILE_DMA() - if (thread_info_flags & _TIF_ASYNC_TLB) { - do_async_page_fault(regs); - return 1; - } + if (thread_info_flags & _TIF_ASYNC_TLB) + do_async_page_fault(regs); #endif - if (thread_info_flags & _TIF_SIGPENDING) { - do_signal(regs); - return 1; - } - if (thread_info_flags & _TIF_NOTIFY_RESUME) { - clear_thread_flag(TIF_NOTIFY_RESUME); - tracehook_notify_resume(regs); - return 1; - } - if (thread_info_flags & _TIF_SINGLESTEP) + + if (thread_info_flags & _TIF_SIGPENDING) + do_signal(regs); + + if (thread_info_flags & _TIF_NOTIFY_RESUME) { + clear_thread_flag(TIF_NOTIFY_RESUME); + tracehook_notify_resume(regs); + } + + local_irq_disable(); + thread_info_flags = READ_ONCE(current_thread_info()->flags); + + } while (thread_info_flags & _TIF_WORK_MASK); + + if (thread_info_flags & _TIF_SINGLESTEP) { single_step_once(regs); +#ifndef __tilegx__ + /* + * FIXME: on tilepro, since we enable interrupts in + * this routine, it's possible that we miss a signal + * or other asynchronous event. + */ + local_irq_disable(); +#endif + } user_enter(); - - return 0; } unsigned long get_wchan(struct task_struct *p) -- cgit v1.2.3 From acfb699e3d757b349251ea19b8173220008d1c9e Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Mon, 21 Dec 2015 13:03:03 -0500 Subject: tile: fix tilepro casts for readl, writel, etc Missing parentheses could cause an argument of the form "integer + pointer" to get cast to "(long)integer + pointer" and remain a pointer type, causing compiler warnings. Signed-off-by: Chris Metcalf --- arch/tile/include/asm/io.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'arch/tile/include') diff --git a/arch/tile/include/asm/io.h b/arch/tile/include/asm/io.h index 322b5fe94781..30f4a210d148 100644 --- a/arch/tile/include/asm/io.h +++ b/arch/tile/include/asm/io.h @@ -161,14 +161,14 @@ extern void _tile_writew(u16 val, unsigned long addr); extern void _tile_writel(u32 val, unsigned long addr); extern void _tile_writeq(u64 val, unsigned long addr); -#define __raw_readb(addr) _tile_readb((unsigned long)addr) -#define __raw_readw(addr) _tile_readw((unsigned long)addr) -#define __raw_readl(addr) _tile_readl((unsigned long)addr) -#define __raw_readq(addr) _tile_readq((unsigned long)addr) -#define __raw_writeb(val, addr) _tile_writeb(val, (unsigned long)addr) -#define __raw_writew(val, addr) _tile_writew(val, (unsigned long)addr) -#define __raw_writel(val, addr) _tile_writel(val, (unsigned long)addr) -#define __raw_writeq(val, addr) _tile_writeq(val, (unsigned long)addr) +#define __raw_readb(addr) _tile_readb((unsigned long)(addr)) +#define __raw_readw(addr) _tile_readw((unsigned long)(addr)) +#define __raw_readl(addr) _tile_readl((unsigned long)(addr)) +#define __raw_readq(addr) _tile_readq((unsigned long)(addr)) +#define __raw_writeb(val, addr) _tile_writeb(val, (unsigned long)(addr)) +#define __raw_writew(val, addr) _tile_writew(val, (unsigned long)(addr)) +#define __raw_writel(val, addr) _tile_writel(val, (unsigned long)(addr)) +#define __raw_writeq(val, addr) _tile_writeq(val, (unsigned long)(addr)) #else /* CONFIG_PCI */ -- cgit v1.2.3 From 00d27c6336b00345724b2510f7c5b8cee3055f02 Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Tue, 5 Jan 2016 11:22:10 -0500 Subject: numa: remove stale node_has_online_mem() define This isn't used anywhere, so delete it. Looks like the last usage (in x86-specific code) was removed by Tejun in 2011 in commit bd6709a91a59 ("x86, NUMA: Make 32bit use common NUMA init path"). Signed-off-by: Chris Metcalf --- arch/tile/include/asm/topology.h | 3 --- include/linux/topology.h | 4 ---- 2 files changed, 7 deletions(-) (limited to 'arch/tile/include') diff --git a/arch/tile/include/asm/topology.h b/arch/tile/include/asm/topology.h index 76b0d0ebb244..b11d5fcd2c41 100644 --- a/arch/tile/include/asm/topology.h +++ b/arch/tile/include/asm/topology.h @@ -44,9 +44,6 @@ static inline const struct cpumask *cpumask_of_node(int node) /* For now, use numa node -1 for global allocation. */ #define pcibus_to_node(bus) ((void)(bus), -1) -/* By definition, we create nodes based on online memory. */ -#define node_has_online_mem(nid) 1 - #endif /* CONFIG_NUMA */ #include diff --git a/include/linux/topology.h b/include/linux/topology.h index 73ddad1e0fa3..afce69296ac0 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -34,10 +34,6 @@ #include #include -#ifndef node_has_online_mem -#define node_has_online_mem(nid) (1) -#endif - #ifndef nr_cpus_node #define nr_cpus_node(node) cpumask_weight(cpumask_of_node(node)) #endif -- cgit v1.2.3