diff options
Diffstat (limited to 'arch/arm/kernel')
-rw-r--r-- | arch/arm/kernel/Makefile | 1 | ||||
-rw-r--r-- | arch/arm/kernel/armksyms.c | 2 | ||||
-rw-r--r-- | arch/arm/kernel/entry-armv.S | 11 | ||||
-rw-r--r-- | arch/arm/kernel/entry-common.S | 65 | ||||
-rw-r--r-- | arch/arm/kernel/etm.c | 15 | ||||
-rw-r--r-- | arch/arm/kernel/ftrace.c | 188 | ||||
-rw-r--r-- | arch/arm/kernel/head-common.S | 2 | ||||
-rw-r--r-- | arch/arm/kernel/head.S | 50 | ||||
-rw-r--r-- | arch/arm/kernel/hw_breakpoint.c | 849 | ||||
-rw-r--r-- | arch/arm/kernel/module.c | 68 | ||||
-rw-r--r-- | arch/arm/kernel/process.c | 24 | ||||
-rw-r--r-- | arch/arm/kernel/ptrace.c | 239 | ||||
-rw-r--r-- | arch/arm/kernel/setup.c | 6 | ||||
-rw-r--r-- | arch/arm/kernel/smp.c | 3 | ||||
-rw-r--r-- | arch/arm/kernel/unwind.c | 2 | ||||
-rw-r--r-- | arch/arm/kernel/vmlinux.lds.S | 15 |
16 files changed, 1435 insertions, 105 deletions
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 980b78e31328..5b9b268f4fbb 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -42,6 +42,7 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_ARM_UNWIND) += unwind.o obj-$(CONFIG_HAVE_TCM) += tcm.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o +obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_CRUNCH) += crunch.o crunch-bits.o AFLAGS_crunch-bits.o := -Wa,-mcpu=ep9312 diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c index 8214bfebfaca..e5e1e5387678 100644 --- a/arch/arm/kernel/armksyms.c +++ b/arch/arm/kernel/armksyms.c @@ -165,6 +165,8 @@ EXPORT_SYMBOL(_find_next_bit_be); #endif #ifdef CONFIG_FUNCTION_TRACER +#ifdef CONFIG_OLD_MCOUNT EXPORT_SYMBOL(mcount); +#endif EXPORT_SYMBOL(__gnu_mcount_nc); #endif diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index bb8e93a76407..c09e3573c5de 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -46,7 +46,8 @@ * this macro assumes that irqstat (r6) and base (r5) are * preserved from get_irqnr_and_base above */ - test_for_ipi r0, r6, r5, lr + ALT_SMP(test_for_ipi r0, r6, r5, lr) + ALT_UP_B(9997f) movne r0, sp adrne lr, BSYM(1b) bne do_IPI @@ -57,6 +58,7 @@ adrne lr, BSYM(1b) bne do_local_timer #endif +9997: #endif .endm @@ -965,11 +967,8 @@ kuser_cmpxchg_fixup: beq 1b rsbs r0, r3, #0 /* beware -- each __kuser slot must be 8 instructions max */ -#ifdef CONFIG_SMP - b __kuser_memory_barrier -#else - usr_ret lr -#endif + ALT_SMP(b __kuser_memory_barrier) + ALT_UP(usr_ret lr) #endif diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 1b560825e1cf..2d23ad985180 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -48,6 +48,8 @@ work_pending: beq no_work_pending mov r0, sp @ 'regs' mov r2, why @ 'syscall' + tst r1, #_TIF_SIGPENDING @ delivering a signal? + movne why, #0 @ prevent further restarts bl do_notify_resume b ret_slow_syscall @ Check work again @@ -127,30 +129,58 @@ ENDPROC(ret_from_fork) * clobber the ip register. This is OK because the ARM calling convention * allows it to be clobbered in subroutines and doesn't use it to hold * parameters.) + * + * When using dynamic ftrace, we patch out the mcount call by a "mov r0, r0" + * for the mcount case, and a "pop {lr}" for the __gnu_mcount_nc case (see + * arch/arm/kernel/ftrace.c). */ + +#ifndef CONFIG_OLD_MCOUNT +#if (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 4)) +#error Ftrace requires CONFIG_FRAME_POINTER=y with GCC older than 4.4.0. +#endif +#endif + #ifdef CONFIG_DYNAMIC_FTRACE -ENTRY(mcount) +ENTRY(__gnu_mcount_nc) + mov ip, lr + ldmia sp!, {lr} + mov pc, ip +ENDPROC(__gnu_mcount_nc) + +ENTRY(ftrace_caller) stmdb sp!, {r0-r3, lr} mov r0, lr sub r0, r0, #MCOUNT_INSN_SIZE + ldr r1, [sp, #20] - .globl mcount_call -mcount_call: + .global ftrace_call +ftrace_call: bl ftrace_stub - ldr lr, [fp, #-4] @ restore lr - ldmia sp!, {r0-r3, pc} + ldmia sp!, {r0-r3, ip, lr} + mov pc, ip +ENDPROC(ftrace_caller) -ENTRY(ftrace_caller) +#ifdef CONFIG_OLD_MCOUNT +ENTRY(mcount) + stmdb sp!, {lr} + ldr lr, [fp, #-4] + ldmia sp!, {pc} +ENDPROC(mcount) + +ENTRY(ftrace_caller_old) stmdb sp!, {r0-r3, lr} ldr r1, [fp, #-4] mov r0, lr sub r0, r0, #MCOUNT_INSN_SIZE - .globl ftrace_call -ftrace_call: + .globl ftrace_call_old +ftrace_call_old: bl ftrace_stub ldr lr, [fp, #-4] @ restore lr ldmia sp!, {r0-r3, pc} +ENDPROC(ftrace_caller_old) +#endif #else @@ -158,7 +188,7 @@ ENTRY(__gnu_mcount_nc) stmdb sp!, {r0-r3, lr} ldr r0, =ftrace_trace_function ldr r2, [r0] - adr r0, ftrace_stub + adr r0, .Lftrace_stub cmp r0, r2 bne gnu_trace ldmia sp!, {r0-r3, ip, lr} @@ -168,11 +198,19 @@ gnu_trace: ldr r1, [sp, #20] @ lr of instrumented routine mov r0, lr sub r0, r0, #MCOUNT_INSN_SIZE - mov lr, pc + adr lr, BSYM(1f) mov pc, r2 +1: ldmia sp!, {r0-r3, ip, lr} mov pc, ip +ENDPROC(__gnu_mcount_nc) +#ifdef CONFIG_OLD_MCOUNT +/* + * This is under an ifdef in order to force link-time errors for people trying + * to build with !FRAME_POINTER with a GCC which doesn't use the new-style + * mcount. + */ ENTRY(mcount) stmdb sp!, {r0-r3, lr} ldr r0, =ftrace_trace_function @@ -191,12 +229,15 @@ trace: mov pc, r2 ldr lr, [fp, #-4] @ restore lr ldmia sp!, {r0-r3, pc} +ENDPROC(mcount) +#endif #endif /* CONFIG_DYNAMIC_FTRACE */ - .globl ftrace_stub -ftrace_stub: +ENTRY(ftrace_stub) +.Lftrace_stub: mov pc, lr +ENDPROC(ftrace_stub) #endif /* CONFIG_FUNCTION_TRACER */ diff --git a/arch/arm/kernel/etm.c b/arch/arm/kernel/etm.c index 33c7077174db..a48d51257988 100644 --- a/arch/arm/kernel/etm.c +++ b/arch/arm/kernel/etm.c @@ -30,6 +30,21 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Alexander Shishkin"); +/* + * ETM tracer state + */ +struct tracectx { + unsigned int etb_bufsz; + void __iomem *etb_regs; + void __iomem *etm_regs; + unsigned long flags; + int ncmppairs; + int etm_portsz; + struct device *dev; + struct clk *emu_clk; + struct mutex mutex; +}; + static struct tracectx tracer; static inline bool trace_isrunning(struct tracectx *t) diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index 0298286ad4ad..971ac8c36ea7 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -2,102 +2,194 @@ * Dynamic function tracing support. * * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> + * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> * * For licencing details, see COPYING. * * Defines low-level handling of mcount calls when the kernel * is compiled with the -pg flag. When using dynamic ftrace, the - * mcount call-sites get patched lazily with NOP till they are - * enabled. All code mutation routines here take effect atomically. + * mcount call-sites get patched with NOP till they are enabled. + * All code mutation routines here are called under stop_machine(). */ #include <linux/ftrace.h> +#include <linux/uaccess.h> #include <asm/cacheflush.h> #include <asm/ftrace.h> -#define PC_OFFSET 8 -#define BL_OPCODE 0xeb000000 -#define BL_OFFSET_MASK 0x00ffffff +#ifdef CONFIG_THUMB2_KERNEL +#define NOP 0xeb04f85d /* pop.w {lr} */ +#else +#define NOP 0xe8bd4000 /* pop {lr} */ +#endif -static unsigned long bl_insn; -static const unsigned long NOP = 0xe1a00000; /* mov r0, r0 */ +#ifdef CONFIG_OLD_MCOUNT +#define OLD_MCOUNT_ADDR ((unsigned long) mcount) +#define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) -unsigned char *ftrace_nop_replace(void) +#define OLD_NOP 0xe1a00000 /* mov r0, r0 */ + +static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) { - return (char *)&NOP; + return rec->arch.old_mcount ? OLD_NOP : NOP; } +static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +{ + if (!rec->arch.old_mcount) + return addr; + + if (addr == MCOUNT_ADDR) + addr = OLD_MCOUNT_ADDR; + else if (addr == FTRACE_ADDR) + addr = OLD_FTRACE_ADDR; + + return addr; +} +#else +static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) +{ + return NOP; +} + +static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +{ + return addr; +} +#endif + /* construct a branch (BL) instruction to addr */ -unsigned char *ftrace_call_replace(unsigned long pc, unsigned long addr) +#ifdef CONFIG_THUMB2_KERNEL +static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) { + unsigned long s, j1, j2, i1, i2, imm10, imm11; + unsigned long first, second; long offset; - offset = (long)addr - (long)(pc + PC_OFFSET); + offset = (long)addr - (long)(pc + 4); + if (offset < -16777216 || offset > 16777214) { + WARN_ON_ONCE(1); + return 0; + } + + s = (offset >> 24) & 0x1; + i1 = (offset >> 23) & 0x1; + i2 = (offset >> 22) & 0x1; + imm10 = (offset >> 12) & 0x3ff; + imm11 = (offset >> 1) & 0x7ff; + + j1 = (!i1) ^ s; + j2 = (!i2) ^ s; + + first = 0xf000 | (s << 10) | imm10; + second = 0xd000 | (j1 << 13) | (j2 << 11) | imm11; + + return (second << 16) | first; +} +#else +static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) +{ + long offset; + + offset = (long)addr - (long)(pc + 8); if (unlikely(offset < -33554432 || offset > 33554428)) { /* Can't generate branches that far (from ARM ARM). Ftrace * doesn't generate branches outside of kernel text. */ WARN_ON_ONCE(1); - return NULL; + return 0; } - offset = (offset >> 2) & BL_OFFSET_MASK; - bl_insn = BL_OPCODE | offset; - return (unsigned char *)&bl_insn; -} -int ftrace_modify_code(unsigned long pc, unsigned char *old_code, - unsigned char *new_code) -{ - unsigned long err = 0, replaced = 0, old, new; + offset = (offset >> 2) & 0x00ffffff; - old = *(unsigned long *)old_code; - new = *(unsigned long *)new_code; + return 0xeb000000 | offset; +} +#endif - __asm__ __volatile__ ( - "1: ldr %1, [%2] \n" - " cmp %1, %4 \n" - "2: streq %3, [%2] \n" - " cmpne %1, %3 \n" - " movne %0, #2 \n" - "3:\n" +static int ftrace_modify_code(unsigned long pc, unsigned long old, + unsigned long new) +{ + unsigned long replaced; - ".pushsection .fixup, \"ax\"\n" - "4: mov %0, #1 \n" - " b 3b \n" - ".popsection\n" + if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) + return -EFAULT; - ".pushsection __ex_table, \"a\"\n" - " .long 1b, 4b \n" - " .long 2b, 4b \n" - ".popsection\n" + if (replaced != old) + return -EINVAL; - : "=r"(err), "=r"(replaced) - : "r"(pc), "r"(new), "r"(old), "0"(err), "1"(replaced) - : "memory"); + if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) + return -EPERM; - if (!err && (replaced == old)) - flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); + flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); - return err; + return 0; } int ftrace_update_ftrace_func(ftrace_func_t func) { - int ret; unsigned long pc, old; - unsigned char *new; + unsigned long new; + int ret; pc = (unsigned long)&ftrace_call; memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE); new = ftrace_call_replace(pc, (unsigned long)func); - ret = ftrace_modify_code(pc, (unsigned char *)&old, new); + + ret = ftrace_modify_code(pc, old, new); + +#ifdef CONFIG_OLD_MCOUNT + if (!ret) { + pc = (unsigned long)&ftrace_call_old; + memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE); + new = ftrace_call_replace(pc, (unsigned long)func); + + ret = ftrace_modify_code(pc, old, new); + } +#endif + + return ret; +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long new, old; + unsigned long ip = rec->ip; + + old = ftrace_nop_replace(rec); + new = ftrace_call_replace(ip, adjust_address(rec, addr)); + + return ftrace_modify_code(rec->ip, old, new); +} + +int ftrace_make_nop(struct module *mod, + struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long ip = rec->ip; + unsigned long old; + unsigned long new; + int ret; + + old = ftrace_call_replace(ip, adjust_address(rec, addr)); + new = ftrace_nop_replace(rec); + ret = ftrace_modify_code(ip, old, new); + +#ifdef CONFIG_OLD_MCOUNT + if (ret == -EINVAL && addr == MCOUNT_ADDR) { + rec->arch.old_mcount = true; + + old = ftrace_call_replace(ip, adjust_address(rec, addr)); + new = ftrace_nop_replace(rec); + ret = ftrace_modify_code(ip, old, new); + } +#endif + return ret; } -/* run from ftrace_init with irqs disabled */ int __init ftrace_dyn_arch_init(void *data) { - ftrace_mcount_set(data); + *(unsigned long *)data = 0; + return 0; } diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S index b9505aa267c0..58a3e632b6d5 100644 --- a/arch/arm/kernel/head-common.S +++ b/arch/arm/kernel/head-common.S @@ -20,7 +20,7 @@ __switch_data: .long __mmap_switched .long __data_loc @ r4 - .long _data @ r5 + .long _sdata @ r5 .long __bss_start @ r6 .long _end @ r7 .long processor_id @ r4 diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index eb62bf947212..b44d21e1e344 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -86,6 +86,9 @@ ENTRY(stext) movs r8, r5 @ invalid machine (r5=0)? beq __error_a @ yes, error 'a' bl __vet_atags +#ifdef CONFIG_SMP_ON_UP + bl __fixup_smp +#endif bl __create_page_tables /* @@ -333,4 +336,51 @@ __create_page_tables: ENDPROC(__create_page_tables) .ltorg +#ifdef CONFIG_SMP_ON_UP +__fixup_smp: + mov r7, #0x00070000 + orr r6, r7, #0xff000000 @ mask 0xff070000 + orr r7, r7, #0x41000000 @ val 0x41070000 + and r0, r9, r6 + teq r0, r7 @ ARM CPU and ARMv6/v7? + bne __fixup_smp_on_up @ no, assume UP + + orr r6, r6, #0x0000ff00 + orr r6, r6, #0x000000f0 @ mask 0xff07fff0 + orr r7, r7, #0x0000b000 + orr r7, r7, #0x00000020 @ val 0x4107b020 + and r0, r9, r6 + teq r0, r7 @ ARM 11MPCore? + moveq pc, lr @ yes, assume SMP + + mrc p15, 0, r0, c0, c0, 5 @ read MPIDR + tst r0, #1 << 31 + movne pc, lr @ bit 31 => SMP + +__fixup_smp_on_up: + adr r0, 1f + ldmia r0, {r3, r6, r7} + sub r3, r0, r3 + add r6, r6, r3 + add r7, r7, r3 +2: cmp r6, r7 + ldmia r6!, {r0, r4} + strlo r4, [r0, r3] + blo 2b + mov pc, lr +ENDPROC(__fixup_smp) + +1: .word . + .word __smpalt_begin + .word __smpalt_end + + .pushsection .data + .globl smp_on_up +smp_on_up: + ALT_SMP(.long 1) + ALT_UP(.long 0) + .popsection + +#endif + #include "head-common.S" diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c new file mode 100644 index 000000000000..54593b0c241b --- /dev/null +++ b/arch/arm/kernel/hw_breakpoint.c @@ -0,0 +1,849 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) 2009, 2010 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + */ + +/* + * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, + * using the CPU's debug registers. + */ +#define pr_fmt(fmt) "hw-breakpoint: " fmt + +#include <linux/errno.h> +#include <linux/perf_event.h> +#include <linux/hw_breakpoint.h> +#include <linux/smp.h> + +#include <asm/cacheflush.h> +#include <asm/cputype.h> +#include <asm/current.h> +#include <asm/hw_breakpoint.h> +#include <asm/kdebug.h> +#include <asm/system.h> +#include <asm/traps.h> + +/* Breakpoint currently in use for each BRP. */ +static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); + +/* Watchpoint currently in use for each WRP. */ +static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); + +/* Number of BRP/WRP registers on this CPU. */ +static int core_num_brps; +static int core_num_wrps; + +/* Debug architecture version. */ +static u8 debug_arch; + +/* Maximum supported watchpoint length. */ +static u8 max_watchpoint_len; + +/* Determine number of BRP registers available. */ +static int get_num_brps(void) +{ + u32 didr; + ARM_DBG_READ(c0, 0, didr); + return ((didr >> 24) & 0xf) + 1; +} + +/* Determine number of WRP registers available. */ +static int get_num_wrps(void) +{ + /* + * FIXME: When a watchpoint fires, the only way to work out which + * watchpoint it was is by disassembling the faulting instruction + * and working out the address of the memory access. + * + * Furthermore, we can only do this if the watchpoint was precise + * since imprecise watchpoints prevent us from calculating register + * based addresses. + * + * For the time being, we only report 1 watchpoint register so we + * always know which watchpoint fired. In the future we can either + * add a disassembler and address generation emulator, or we can + * insert a check to see if the DFAR is set on watchpoint exception + * entry [the ARM ARM states that the DFAR is UNKNOWN, but + * experience shows that it is set on some implementations]. + */ + +#if 0 + u32 didr, wrps; + ARM_DBG_READ(c0, 0, didr); + return ((didr >> 28) & 0xf) + 1; +#endif + + return 1; +} + +int hw_breakpoint_slots(int type) +{ + /* + * We can be called early, so don't rely on + * our static variables being initialised. + */ + switch (type) { + case TYPE_INST: + return get_num_brps(); + case TYPE_DATA: + return get_num_wrps(); + default: + pr_warning("unknown slot type: %d\n", type); + return 0; + } +} + +/* Determine debug architecture. */ +static u8 get_debug_arch(void) +{ + u32 didr; + + /* Do we implement the extended CPUID interface? */ + if (((read_cpuid_id() >> 16) & 0xf) != 0xf) { + pr_warning("CPUID feature registers not supported. " + "Assuming v6 debug is present.\n"); + return ARM_DEBUG_ARCH_V6; + } + + ARM_DBG_READ(c0, 0, didr); + return (didr >> 16) & 0xf; +} + +/* Does this core support mismatch breakpoints? */ +static int core_has_mismatch_bps(void) +{ + return debug_arch >= ARM_DEBUG_ARCH_V7_ECP14 && core_num_brps > 1; +} + +u8 arch_get_debug_arch(void) +{ + return debug_arch; +} + +#define READ_WB_REG_CASE(OP2, M, VAL) \ + case ((OP2 << 4) + M): \ + ARM_DBG_READ(c ## M, OP2, VAL); \ + break + +#define WRITE_WB_REG_CASE(OP2, M, VAL) \ + case ((OP2 << 4) + M): \ + ARM_DBG_WRITE(c ## M, OP2, VAL);\ + break + +#define GEN_READ_WB_REG_CASES(OP2, VAL) \ + READ_WB_REG_CASE(OP2, 0, VAL); \ + READ_WB_REG_CASE(OP2, 1, VAL); \ + READ_WB_REG_CASE(OP2, 2, VAL); \ + READ_WB_REG_CASE(OP2, 3, VAL); \ + READ_WB_REG_CASE(OP2, 4, VAL); \ + READ_WB_REG_CASE(OP2, 5, VAL); \ + READ_WB_REG_CASE(OP2, 6, VAL); \ + READ_WB_REG_CASE(OP2, 7, VAL); \ + READ_WB_REG_CASE(OP2, 8, VAL); \ + READ_WB_REG_CASE(OP2, 9, VAL); \ + READ_WB_REG_CASE(OP2, 10, VAL); \ + READ_WB_REG_CASE(OP2, 11, VAL); \ + READ_WB_REG_CASE(OP2, 12, VAL); \ + READ_WB_REG_CASE(OP2, 13, VAL); \ + READ_WB_REG_CASE(OP2, 14, VAL); \ + READ_WB_REG_CASE(OP2, 15, VAL) + +#define GEN_WRITE_WB_REG_CASES(OP2, VAL) \ + WRITE_WB_REG_CASE(OP2, 0, VAL); \ + WRITE_WB_REG_CASE(OP2, 1, VAL); \ + WRITE_WB_REG_CASE(OP2, 2, VAL); \ + WRITE_WB_REG_CASE(OP2, 3, VAL); \ + WRITE_WB_REG_CASE(OP2, 4, VAL); \ + WRITE_WB_REG_CASE(OP2, 5, VAL); \ + WRITE_WB_REG_CASE(OP2, 6, VAL); \ + WRITE_WB_REG_CASE(OP2, 7, VAL); \ + WRITE_WB_REG_CASE(OP2, 8, VAL); \ + WRITE_WB_REG_CASE(OP2, 9, VAL); \ + WRITE_WB_REG_CASE(OP2, 10, VAL); \ + WRITE_WB_REG_CASE(OP2, 11, VAL); \ + WRITE_WB_REG_CASE(OP2, 12, VAL); \ + WRITE_WB_REG_CASE(OP2, 13, VAL); \ + WRITE_WB_REG_CASE(OP2, 14, VAL); \ + WRITE_WB_REG_CASE(OP2, 15, VAL) + +static u32 read_wb_reg(int n) +{ + u32 val = 0; + + switch (n) { + GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val); + GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val); + GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val); + GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val); + default: + pr_warning("attempt to read from unknown breakpoint " + "register %d\n", n); + } + + return val; +} + +static void write_wb_reg(int n, u32 val) +{ + switch (n) { + GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val); + GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val); + GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val); + GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val); + default: + pr_warning("attempt to write to unknown breakpoint " + "register %d\n", n); + } + isb(); +} + +/* + * In order to access the breakpoint/watchpoint control registers, + * we must be running in debug monitor mode. Unfortunately, we can + * be put into halting debug mode at any time by an external debugger + * but there is nothing we can do to prevent that. + */ +static int enable_monitor_mode(void) +{ + u32 dscr; + int ret = 0; + + ARM_DBG_READ(c1, 0, dscr); + + /* Ensure that halting mode is disabled. */ + if (WARN_ONCE(dscr & ARM_DSCR_HDBGEN, "halting debug mode enabled." + "Unable to access hardware resources.")) { + ret = -EPERM; + goto out; + } + + /* Write to the corresponding DSCR. */ + switch (debug_arch) { + case ARM_DEBUG_ARCH_V6: + case ARM_DEBUG_ARCH_V6_1: + ARM_DBG_WRITE(c1, 0, (dscr | ARM_DSCR_MDBGEN)); + break; + case ARM_DEBUG_ARCH_V7_ECP14: + ARM_DBG_WRITE(c2, 2, (dscr | ARM_DSCR_MDBGEN)); + break; + default: + ret = -ENODEV; + goto out; + } + + /* Check that the write made it through. */ + ARM_DBG_READ(c1, 0, dscr); + if (WARN_ONCE(!(dscr & ARM_DSCR_MDBGEN), + "failed to enable monitor mode.")) { + ret = -EPERM; + } + +out: + return ret; +} + +/* + * Check if 8-bit byte-address select is available. + * This clobbers WRP 0. + */ +static u8 get_max_wp_len(void) +{ + u32 ctrl_reg; + struct arch_hw_breakpoint_ctrl ctrl; + u8 size = 4; + + if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14) + goto out; + + if (enable_monitor_mode()) + goto out; + + memset(&ctrl, 0, sizeof(ctrl)); + ctrl.len = ARM_BREAKPOINT_LEN_8; + ctrl_reg = encode_ctrl_reg(ctrl); + + write_wb_reg(ARM_BASE_WVR, 0); + write_wb_reg(ARM_BASE_WCR, ctrl_reg); + if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg) + size = 8; + +out: + return size; +} + +u8 arch_get_max_wp_len(void) +{ + return max_watchpoint_len; +} + +/* + * Handler for reactivating a suspended watchpoint when the single + * step `mismatch' breakpoint is triggered. + */ +static void wp_single_step_handler(struct perf_event *bp, int unused, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + perf_event_enable(counter_arch_bp(bp)->suspended_wp); + unregister_hw_breakpoint(bp); +} + +static int bp_is_single_step(struct perf_event *bp) +{ + return bp->overflow_handler == wp_single_step_handler; +} + +/* + * Install a perf counter breakpoint. + */ +int arch_install_hw_breakpoint(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + struct perf_event **slot, **slots; + int i, max_slots, ctrl_base, val_base, ret = 0; + + /* Ensure that we are in monitor mode and halting mode is disabled. */ + ret = enable_monitor_mode(); + if (ret) + goto out; + + if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { + /* Breakpoint */ + ctrl_base = ARM_BASE_BCR; + val_base = ARM_BASE_BVR; + slots = __get_cpu_var(bp_on_reg); + max_slots = core_num_brps - 1; + + if (bp_is_single_step(bp)) { + info->ctrl.mismatch = 1; + i = max_slots; + slots[i] = bp; + goto setup; + } + } else { + /* Watchpoint */ + ctrl_base = ARM_BASE_WCR; + val_base = ARM_BASE_WVR; + slots = __get_cpu_var(wp_on_reg); + max_slots = core_num_wrps; + } + + for (i = 0; i < max_slots; ++i) { + slot = &slots[i]; + + if (!*slot) { + *slot = bp; + break; + } + } + + if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot")) { + ret = -EBUSY; + goto out; + } + +setup: + /* Setup the address register. */ + write_wb_reg(val_base + i, info->address); + + /* Setup the control register. */ + write_wb_reg(ctrl_base + i, encode_ctrl_reg(info->ctrl) | 0x1); + +out: + return ret; +} + +void arch_uninstall_hw_breakpoint(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + struct perf_event **slot, **slots; + int i, max_slots, base; + + if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { + /* Breakpoint */ + base = ARM_BASE_BCR; + slots = __get_cpu_var(bp_on_reg); + max_slots = core_num_brps - 1; + + if (bp_is_single_step(bp)) { + i = max_slots; + slots[i] = NULL; + goto reset; + } + } else { + /* Watchpoint */ + base = ARM_BASE_WCR; + slots = __get_cpu_var(wp_on_reg); + max_slots = core_num_wrps; + } + + /* Remove the breakpoint. */ + for (i = 0; i < max_slots; ++i) { + slot = &slots[i]; + + if (*slot == bp) { + *slot = NULL; + break; + } + } + + if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot")) + return; + +reset: + /* Reset the control register. */ + write_wb_reg(base + i, 0); +} + +static int get_hbp_len(u8 hbp_len) +{ + unsigned int len_in_bytes = 0; + + switch (hbp_len) { + case ARM_BREAKPOINT_LEN_1: + len_in_bytes = 1; + break; + case ARM_BREAKPOINT_LEN_2: + len_in_bytes = 2; + break; + case ARM_BREAKPOINT_LEN_4: + len_in_bytes = 4; + break; + case ARM_BREAKPOINT_LEN_8: + len_in_bytes = 8; + break; + } + + return len_in_bytes; +} + +/* + * Check whether bp virtual address is in kernel space. + */ +int arch_check_bp_in_kernelspace(struct perf_event *bp) +{ + unsigned int len; + unsigned long va; + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + + va = info->address; + len = get_hbp_len(info->ctrl.len); + + return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); +} + +/* + * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. + * Hopefully this will disappear when ptrace can bypass the conversion + * to generic breakpoint descriptions. + */ +int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, + int *gen_len, int *gen_type) +{ + /* Type */ + switch (ctrl.type) { + case ARM_BREAKPOINT_EXECUTE: + *gen_type = HW_BREAKPOINT_X; + break; + case ARM_BREAKPOINT_LOAD: + *gen_type = HW_BREAKPOINT_R; + break; + case ARM_BREAKPOINT_STORE: + *gen_type = HW_BREAKPOINT_W; + break; + case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: + *gen_type = HW_BREAKPOINT_RW; + break; + default: + return -EINVAL; + } + + /* Len */ + switch (ctrl.len) { + case ARM_BREAKPOINT_LEN_1: + *gen_len = HW_BREAKPOINT_LEN_1; + break; + case ARM_BREAKPOINT_LEN_2: + *gen_len = HW_BREAKPOINT_LEN_2; + break; + case ARM_BREAKPOINT_LEN_4: + *gen_len = HW_BREAKPOINT_LEN_4; + break; + case ARM_BREAKPOINT_LEN_8: + *gen_len = HW_BREAKPOINT_LEN_8; + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * Construct an arch_hw_breakpoint from a perf_event. + */ +static int arch_build_bp_info(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + + /* Type */ + switch (bp->attr.bp_type) { + case HW_BREAKPOINT_X: + info->ctrl.type = ARM_BREAKPOINT_EXECUTE; + break; + case HW_BREAKPOINT_R: + info->ctrl.type = ARM_BREAKPOINT_LOAD; + break; + case HW_BREAKPOINT_W: + info->ctrl.type = ARM_BREAKPOINT_STORE; + break; + case HW_BREAKPOINT_RW: + info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; + break; + default: + return -EINVAL; + } + + /* Len */ + switch (bp->attr.bp_len) { + case HW_BREAKPOINT_LEN_1: + info->ctrl.len = ARM_BREAKPOINT_LEN_1; + break; + case HW_BREAKPOINT_LEN_2: + info->ctrl.len = ARM_BREAKPOINT_LEN_2; + break; + case HW_BREAKPOINT_LEN_4: + info->ctrl.len = ARM_BREAKPOINT_LEN_4; + break; + case HW_BREAKPOINT_LEN_8: + info->ctrl.len = ARM_BREAKPOINT_LEN_8; + if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE) + && max_watchpoint_len >= 8) + break; + default: + return -EINVAL; + } + + /* Address */ + info->address = bp->attr.bp_addr; + + /* Privilege */ + info->ctrl.privilege = ARM_BREAKPOINT_USER; + if (arch_check_bp_in_kernelspace(bp) && !bp_is_single_step(bp)) + info->ctrl.privilege |= ARM_BREAKPOINT_PRIV; + + /* Enabled? */ + info->ctrl.enabled = !bp->attr.disabled; + + /* Mismatch */ + info->ctrl.mismatch = 0; + + return 0; +} + +/* + * Validate the arch-specific HW Breakpoint register settings. + */ +int arch_validate_hwbkpt_settings(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + int ret = 0; + u32 bytelen, max_len, offset, alignment_mask = 0x3; + + /* Build the arch_hw_breakpoint. */ + ret = arch_build_bp_info(bp); + if (ret) + goto out; + + /* Check address alignment. */ + if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) + alignment_mask = 0x7; + if (info->address & alignment_mask) { + /* + * Try to fix the alignment. This may result in a length + * that is too large, so we must check for that. + */ + bytelen = get_hbp_len(info->ctrl.len); + max_len = info->ctrl.type == ARM_BREAKPOINT_EXECUTE ? 4 : + max_watchpoint_len; + + if (max_len >= 8) + offset = info->address & 0x7; + else + offset = info->address & 0x3; + + if (bytelen > (1 << ((max_len - (offset + 1)) >> 1))) { + ret = -EFBIG; + goto out; + } + + info->ctrl.len <<= offset; + info->address &= ~offset; + + pr_debug("breakpoint alignment fixup: length = 0x%x, " + "address = 0x%x\n", info->ctrl.len, info->address); + } + + /* + * Currently we rely on an overflow handler to take + * care of single-stepping the breakpoint when it fires. + * In the case of userspace breakpoints on a core with V7 debug, + * we can use the mismatch feature as a poor-man's hardware single-step. + */ + if (WARN_ONCE(!bp->overflow_handler && + (arch_check_bp_in_kernelspace(bp) || !core_has_mismatch_bps()), + "overflow handler required but none found")) { + ret = -EINVAL; + goto out; + } +out: + return ret; +} + +static void update_mismatch_flag(int idx, int flag) +{ + struct perf_event *bp = __get_cpu_var(bp_on_reg[idx]); + struct arch_hw_breakpoint *info; + + if (bp == NULL) + return; + + info = counter_arch_bp(bp); + + /* Update the mismatch field to enter/exit `single-step' mode */ + if (!bp->overflow_handler && info->ctrl.mismatch != flag) { + info->ctrl.mismatch = flag; + write_wb_reg(ARM_BASE_BCR + idx, encode_ctrl_reg(info->ctrl) | 0x1); + } +} + +static void watchpoint_handler(unsigned long unknown, struct pt_regs *regs) +{ + int i; + struct perf_event *bp, **slots = __get_cpu_var(wp_on_reg); + struct arch_hw_breakpoint *info; + struct perf_event_attr attr; + + /* Without a disassembler, we can only handle 1 watchpoint. */ + BUG_ON(core_num_wrps > 1); + + hw_breakpoint_init(&attr); + attr.bp_addr = regs->ARM_pc & ~0x3; + attr.bp_len = HW_BREAKPOINT_LEN_4; + attr.bp_type = HW_BREAKPOINT_X; + + for (i = 0; i < core_num_wrps; ++i) { + rcu_read_lock(); + + if (slots[i] == NULL) { + rcu_read_unlock(); + continue; + } + + /* + * The DFAR is an unknown value. Since we only allow a + * single watchpoint, we can set the trigger to the lowest + * possible faulting address. + */ + info = counter_arch_bp(slots[i]); + info->trigger = slots[i]->attr.bp_addr; + pr_debug("watchpoint fired: address = 0x%x\n", info->trigger); + perf_bp_event(slots[i], regs); + + /* + * If no overflow handler is present, insert a temporary + * mismatch breakpoint so we can single-step over the + * watchpoint trigger. + */ + if (!slots[i]->overflow_handler) { + bp = register_user_hw_breakpoint(&attr, + wp_single_step_handler, + current); + counter_arch_bp(bp)->suspended_wp = slots[i]; + perf_event_disable(slots[i]); + } + + rcu_read_unlock(); + } +} + +static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs) +{ + int i; + int mismatch; + u32 ctrl_reg, val, addr; + struct perf_event *bp, **slots = __get_cpu_var(bp_on_reg); + struct arch_hw_breakpoint *info; + struct arch_hw_breakpoint_ctrl ctrl; + + /* The exception entry code places the amended lr in the PC. */ + addr = regs->ARM_pc; + + for (i = 0; i < core_num_brps; ++i) { + rcu_read_lock(); + + bp = slots[i]; + + if (bp == NULL) { + rcu_read_unlock(); + continue; + } + + mismatch = 0; + + /* Check if the breakpoint value matches. */ + val = read_wb_reg(ARM_BASE_BVR + i); + if (val != (addr & ~0x3)) + goto unlock; + + /* Possible match, check the byte address select to confirm. */ + ctrl_reg = read_wb_reg(ARM_BASE_BCR + i); + decode_ctrl_reg(ctrl_reg, &ctrl); + if ((1 << (addr & 0x3)) & ctrl.len) { + mismatch = 1; + info = counter_arch_bp(bp); + info->trigger = addr; + } + +unlock: + if ((mismatch && !info->ctrl.mismatch) || bp_is_single_step(bp)) { + pr_debug("breakpoint fired: address = 0x%x\n", addr); + perf_bp_event(bp, regs); + } + + update_mismatch_flag(i, mismatch); + rcu_read_unlock(); + } +} + +/* + * Called from either the Data Abort Handler [watchpoint] or the + * Prefetch Abort Handler [breakpoint]. + */ +static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, + struct pt_regs *regs) +{ + int ret = 1; /* Unhandled fault. */ + u32 dscr; + + /* We only handle watchpoints and hardware breakpoints. */ + ARM_DBG_READ(c1, 0, dscr); + + /* Perform perf callbacks. */ + switch (ARM_DSCR_MOE(dscr)) { + case ARM_ENTRY_BREAKPOINT: + breakpoint_handler(addr, regs); + break; + case ARM_ENTRY_ASYNC_WATCHPOINT: + WARN_ON("Asynchronous watchpoint exception taken. " + "Debugging results may be unreliable"); + case ARM_ENTRY_SYNC_WATCHPOINT: + watchpoint_handler(addr, regs); + break; + default: + goto out; + } + + ret = 0; +out: + return ret; +} + +/* + * One-time initialisation. + */ +static void __init reset_ctrl_regs(void *unused) +{ + int i; + + if (enable_monitor_mode()) + return; + + for (i = 0; i < core_num_brps; ++i) { + write_wb_reg(ARM_BASE_BCR + i, 0UL); + write_wb_reg(ARM_BASE_BVR + i, 0UL); + } + + for (i = 0; i < core_num_wrps; ++i) { + write_wb_reg(ARM_BASE_WCR + i, 0UL); + write_wb_reg(ARM_BASE_WVR + i, 0UL); + } +} + +static int __init arch_hw_breakpoint_init(void) +{ + int ret = 0; + u32 dscr; + + debug_arch = get_debug_arch(); + + if (debug_arch > ARM_DEBUG_ARCH_V7_ECP14) { + pr_info("debug architecture 0x%x unsupported.\n", debug_arch); + ret = -ENODEV; + goto out; + } + + /* Determine how many BRPs/WRPs are available. */ + core_num_brps = get_num_brps(); + core_num_wrps = get_num_wrps(); + + pr_info("found %d breakpoint and %d watchpoint registers.\n", + core_num_brps, core_num_wrps); + + if (core_has_mismatch_bps()) + pr_info("1 breakpoint reserved for watchpoint single-step.\n"); + + ARM_DBG_READ(c1, 0, dscr); + if (dscr & ARM_DSCR_HDBGEN) { + pr_warning("halting debug mode enabled. Assuming maximum " + "watchpoint size of 4 bytes."); + } else { + /* Work out the maximum supported watchpoint length. */ + max_watchpoint_len = get_max_wp_len(); + pr_info("maximum watchpoint size is %u bytes.\n", + max_watchpoint_len); + + /* + * Reset the breakpoint resources. We assume that a halting + * debugger will leave the world in a nice state for us. + */ + smp_call_function(reset_ctrl_regs, NULL, 1); + reset_ctrl_regs(NULL); + } + + /* Register debug fault handler. */ + hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, + "watchpoint debug exception"); + hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, + "breakpoint debug exception"); + +out: + return ret; +} +arch_initcall(arch_hw_breakpoint_init); + +void hw_breakpoint_pmu_read(struct perf_event *bp) +{ +} + +/* + * Dummy function to register with die_notifier. + */ +int hw_breakpoint_exceptions_notify(struct notifier_block *unused, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index 6b4605893f1e..d9bd786ce23d 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -69,20 +69,31 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, { #ifdef CONFIG_ARM_UNWIND Elf_Shdr *s, *sechdrs_end = sechdrs + hdr->e_shnum; + struct arm_unwind_mapping *maps = mod->arch.map; for (s = sechdrs; s < sechdrs_end; s++) { - if (strcmp(".ARM.exidx.init.text", secstrings + s->sh_name) == 0) - mod->arch.unw_sec_init = s; - else if (strcmp(".ARM.exidx.devinit.text", secstrings + s->sh_name) == 0) - mod->arch.unw_sec_devinit = s; - else if (strcmp(".ARM.exidx", secstrings + s->sh_name) == 0) - mod->arch.unw_sec_core = s; - else if (strcmp(".init.text", secstrings + s->sh_name) == 0) - mod->arch.sec_init_text = s; - else if (strcmp(".devinit.text", secstrings + s->sh_name) == 0) - mod->arch.sec_devinit_text = s; - else if (strcmp(".text", secstrings + s->sh_name) == 0) - mod->arch.sec_core_text = s; + char const *secname = secstrings + s->sh_name; + + if (strcmp(".ARM.exidx.init.text", secname) == 0) + maps[ARM_SEC_INIT].unw_sec = s; + else if (strcmp(".ARM.exidx.devinit.text", secname) == 0) + maps[ARM_SEC_DEVINIT].unw_sec = s; + else if (strcmp(".ARM.exidx", secname) == 0) + maps[ARM_SEC_CORE].unw_sec = s; + else if (strcmp(".ARM.exidx.exit.text", secname) == 0) + maps[ARM_SEC_EXIT].unw_sec = s; + else if (strcmp(".ARM.exidx.devexit.text", secname) == 0) + maps[ARM_SEC_DEVEXIT].unw_sec = s; + else if (strcmp(".init.text", secname) == 0) + maps[ARM_SEC_INIT].sec_text = s; + else if (strcmp(".devinit.text", secname) == 0) + maps[ARM_SEC_DEVINIT].sec_text = s; + else if (strcmp(".text", secname) == 0) + maps[ARM_SEC_CORE].sec_text = s; + else if (strcmp(".exit.text", secname) == 0) + maps[ARM_SEC_EXIT].sec_text = s; + else if (strcmp(".devexit.text", secname) == 0) + maps[ARM_SEC_DEVEXIT].sec_text = s; } #endif return 0; @@ -292,31 +303,22 @@ apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, #ifdef CONFIG_ARM_UNWIND static void register_unwind_tables(struct module *mod) { - if (mod->arch.unw_sec_init && mod->arch.sec_init_text) - mod->arch.unwind_init = - unwind_table_add(mod->arch.unw_sec_init->sh_addr, - mod->arch.unw_sec_init->sh_size, - mod->arch.sec_init_text->sh_addr, - mod->arch.sec_init_text->sh_size); - if (mod->arch.unw_sec_devinit && mod->arch.sec_devinit_text) - mod->arch.unwind_devinit = - unwind_table_add(mod->arch.unw_sec_devinit->sh_addr, - mod->arch.unw_sec_devinit->sh_size, - mod->arch.sec_devinit_text->sh_addr, - mod->arch.sec_devinit_text->sh_size); - if (mod->arch.unw_sec_core && mod->arch.sec_core_text) - mod->arch.unwind_core = - unwind_table_add(mod->arch.unw_sec_core->sh_addr, - mod->arch.unw_sec_core->sh_size, - mod->arch.sec_core_text->sh_addr, - mod->arch.sec_core_text->sh_size); + int i; + for (i = 0; i < ARM_SEC_MAX; ++i) { + struct arm_unwind_mapping *map = &mod->arch.map[i]; + if (map->unw_sec && map->sec_text) + map->unwind = unwind_table_add(map->unw_sec->sh_addr, + map->unw_sec->sh_size, + map->sec_text->sh_addr, + map->sec_text->sh_size); + } } static void unregister_unwind_tables(struct module *mod) { - unwind_table_del(mod->arch.unwind_init); - unwind_table_del(mod->arch.unwind_devinit); - unwind_table_del(mod->arch.unwind_core); + int i = ARM_SEC_MAX; + while (--i >= 0) + unwind_table_del(mod->arch.map[i].unwind); } #else static inline void register_unwind_tables(struct module *mod) { } diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 401e38be1f78..3af34bf4f4df 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -29,6 +29,7 @@ #include <linux/utsname.h> #include <linux/uaccess.h> #include <linux/random.h> +#include <linux/hw_breakpoint.h> #include <asm/cacheflush.h> #include <asm/leds.h> @@ -135,6 +136,25 @@ EXPORT_SYMBOL(pm_power_off); void (*arm_pm_restart)(char str, const char *cmd) = arm_machine_restart; EXPORT_SYMBOL_GPL(arm_pm_restart); +static void do_nothing(void *unused) +{ +} + +/* + * cpu_idle_wait - Used to ensure that all the CPUs discard old value of + * pm_idle and update to new pm_idle value. Required while changing pm_idle + * handler on SMP systems. + * + * Caller must have changed pm_idle to the new value before the call. Old + * pm_idle value will not be used by any CPU after the return of this function. + */ +void cpu_idle_wait(void) +{ + smp_mb(); + /* kick all the CPUs so that they exit out of pm_idle */ + smp_call_function(do_nothing, NULL, 1); +} +EXPORT_SYMBOL_GPL(cpu_idle_wait); /* * This is our default idle handler. We need to disable @@ -317,6 +337,8 @@ void flush_thread(void) struct thread_info *thread = current_thread_info(); struct task_struct *tsk = current; + flush_ptrace_hw_breakpoint(tsk); + memset(thread->used_cp, 0, sizeof(thread->used_cp)); memset(&tsk->thread.debug, 0, sizeof(struct debug_info)); memset(&thread->fpstate, 0, sizeof(union fp_state)); @@ -345,6 +367,8 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, thread->cpu_context.sp = (unsigned long)childregs; thread->cpu_context.pc = (unsigned long)ret_from_fork; + clear_ptrace_hw_breakpoint(p); + if (clone_flags & CLONE_SETTLS) thread->tp_value = regs->ARM_r3; diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index f99d489822d5..e0cb6370ed14 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -19,6 +19,8 @@ #include <linux/init.h> #include <linux/signal.h> #include <linux/uaccess.h> +#include <linux/perf_event.h> +#include <linux/hw_breakpoint.h> #include <asm/pgtable.h> #include <asm/system.h> @@ -847,6 +849,232 @@ static int ptrace_setvfpregs(struct task_struct *tsk, void __user *data) } #endif +#ifdef CONFIG_HAVE_HW_BREAKPOINT +/* + * Convert a virtual register number into an index for a thread_info + * breakpoint array. Breakpoints are identified using positive numbers + * whilst watchpoints are negative. The registers are laid out as pairs + * of (address, control), each pair mapping to a unique hw_breakpoint struct. + * Register 0 is reserved for describing resource information. + */ +static int ptrace_hbp_num_to_idx(long num) +{ + if (num < 0) + num = (ARM_MAX_BRP << 1) - num; + return (num - 1) >> 1; +} + +/* + * Returns the virtual register number for the address of the + * breakpoint at index idx. + */ +static long ptrace_hbp_idx_to_num(int idx) +{ + long mid = ARM_MAX_BRP << 1; + long num = (idx << 1) + 1; + return num > mid ? mid - num : num; +} + +/* + * Handle hitting a HW-breakpoint. + */ +static void ptrace_hbptriggered(struct perf_event *bp, int unused, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp); + long num; + int i; + siginfo_t info; + + for (i = 0; i < ARM_MAX_HBP_SLOTS; ++i) + if (current->thread.debug.hbp[i] == bp) + break; + + num = (i == ARM_MAX_HBP_SLOTS) ? 0 : ptrace_hbp_idx_to_num(i); + + info.si_signo = SIGTRAP; + info.si_errno = (int)num; + info.si_code = TRAP_HWBKPT; + info.si_addr = (void __user *)(bkpt->trigger); + + force_sig_info(SIGTRAP, &info, current); +} + +/* + * Set ptrace breakpoint pointers to zero for this task. + * This is required in order to prevent child processes from unregistering + * breakpoints held by their parent. + */ +void clear_ptrace_hw_breakpoint(struct task_struct *tsk) +{ + memset(tsk->thread.debug.hbp, 0, sizeof(tsk->thread.debug.hbp)); +} + +/* + * Unregister breakpoints from this task and reset the pointers in + * the thread_struct. + */ +void flush_ptrace_hw_breakpoint(struct task_struct *tsk) +{ + int i; + struct thread_struct *t = &tsk->thread; + + for (i = 0; i < ARM_MAX_HBP_SLOTS; i++) { + if (t->debug.hbp[i]) { + unregister_hw_breakpoint(t->debug.hbp[i]); + t->debug.hbp[i] = NULL; + } + } +} + +static u32 ptrace_get_hbp_resource_info(void) +{ + u8 num_brps, num_wrps, debug_arch, wp_len; + u32 reg = 0; + + num_brps = hw_breakpoint_slots(TYPE_INST); + num_wrps = hw_breakpoint_slots(TYPE_DATA); + debug_arch = arch_get_debug_arch(); + wp_len = arch_get_max_wp_len(); + + reg |= debug_arch; + reg <<= 8; + reg |= wp_len; + reg <<= 8; + reg |= num_wrps; + reg <<= 8; + reg |= num_brps; + + return reg; +} + +static struct perf_event *ptrace_hbp_create(struct task_struct *tsk, int type) +{ + struct perf_event_attr attr; + + ptrace_breakpoint_init(&attr); + + /* Initialise fields to sane defaults. */ + attr.bp_addr = 0; + attr.bp_len = HW_BREAKPOINT_LEN_4; + attr.bp_type = type; + attr.disabled = 1; + + return register_user_hw_breakpoint(&attr, ptrace_hbptriggered, tsk); +} + +static int ptrace_gethbpregs(struct task_struct *tsk, long num, + unsigned long __user *data) +{ + u32 reg; + int idx, ret = 0; + struct perf_event *bp; + struct arch_hw_breakpoint_ctrl arch_ctrl; + + if (num == 0) { + reg = ptrace_get_hbp_resource_info(); + } else { + idx = ptrace_hbp_num_to_idx(num); + if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) { + ret = -EINVAL; + goto out; + } + + bp = tsk->thread.debug.hbp[idx]; + if (!bp) { + reg = 0; + goto put; + } + + arch_ctrl = counter_arch_bp(bp)->ctrl; + + /* + * Fix up the len because we may have adjusted it + * to compensate for an unaligned address. + */ + while (!(arch_ctrl.len & 0x1)) + arch_ctrl.len >>= 1; + + if (idx & 0x1) + reg = encode_ctrl_reg(arch_ctrl); + else + reg = bp->attr.bp_addr; + } + +put: + if (put_user(reg, data)) + ret = -EFAULT; + +out: + return ret; +} + +static int ptrace_sethbpregs(struct task_struct *tsk, long num, + unsigned long __user *data) +{ + int idx, gen_len, gen_type, implied_type, ret = 0; + u32 user_val; + struct perf_event *bp; + struct arch_hw_breakpoint_ctrl ctrl; + struct perf_event_attr attr; + + if (num == 0) + goto out; + else if (num < 0) + implied_type = HW_BREAKPOINT_RW; + else + implied_type = HW_BREAKPOINT_X; + + idx = ptrace_hbp_num_to_idx(num); + if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) { + ret = -EINVAL; + goto out; + } + + if (get_user(user_val, data)) { + ret = -EFAULT; + goto out; + } + + bp = tsk->thread.debug.hbp[idx]; + if (!bp) { + bp = ptrace_hbp_create(tsk, implied_type); + if (IS_ERR(bp)) { + ret = PTR_ERR(bp); + goto out; + } + tsk->thread.debug.hbp[idx] = bp; + } + + attr = bp->attr; + + if (num & 0x1) { + /* Address */ + attr.bp_addr = user_val; + } else { + /* Control */ + decode_ctrl_reg(user_val, &ctrl); + ret = arch_bp_generic_fields(ctrl, &gen_len, &gen_type); + if (ret) + goto out; + + if ((gen_type & implied_type) != gen_type) { + ret = -EINVAL; + goto out; + } + + attr.bp_len = gen_len; + attr.bp_type = gen_type; + attr.disabled = !ctrl.enabled; + } + + ret = modify_user_hw_breakpoint(bp, &attr); +out: + return ret; +} +#endif + long arch_ptrace(struct task_struct *child, long request, long addr, long data) { int ret; @@ -916,6 +1144,17 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data) break; #endif +#ifdef CONFIG_HAVE_HW_BREAKPOINT + case PTRACE_GETHBPREGS: + ret = ptrace_gethbpregs(child, addr, + (unsigned long __user *)data); + break; + case PTRACE_SETHBPREGS: + ret = ptrace_sethbpregs(child, addr, + (unsigned long __user *)data); + break; +#endif + default: ret = ptrace_request(child, request, addr, data); break; diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index e0430d036cea..336f14e0e5c2 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -36,6 +36,7 @@ #include <asm/procinfo.h> #include <asm/sections.h> #include <asm/setup.h> +#include <asm/smp_plat.h> #include <asm/mach-types.h> #include <asm/cacheflush.h> #include <asm/cachetype.h> @@ -524,7 +525,7 @@ request_standard_resources(struct meminfo *mi, struct machine_desc *mdesc) kernel_code.start = virt_to_phys(_text); kernel_code.end = virt_to_phys(_etext - 1); - kernel_data.start = virt_to_phys(_data); + kernel_data.start = virt_to_phys(_sdata); kernel_data.end = virt_to_phys(_end - 1); for (i = 0; i < mi->nr_banks; i++) { @@ -859,7 +860,8 @@ void __init setup_arch(char **cmdline_p) request_standard_resources(&meminfo, mdesc); #ifdef CONFIG_SMP - smp_init_cpus(); + if (is_smp()) + smp_init_cpus(); #endif reserve_crashkernel(); diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 40dc74f2b27f..32e16da5cbce 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -567,7 +567,8 @@ void smp_send_stop(void) { cpumask_t mask = cpu_online_map; cpu_clear(smp_processor_id(), mask); - send_ipi_message(&mask, IPI_CPU_STOP); + if (!cpus_empty(mask)) + send_ipi_message(&mask, IPI_CPU_STOP); } /* diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c index dd81a918c106..2a161765f6d5 100644 --- a/arch/arm/kernel/unwind.c +++ b/arch/arm/kernel/unwind.c @@ -146,6 +146,8 @@ static struct unwind_idx *unwind_find_idx(unsigned long addr) addr < table->end_addr) { idx = search_index(addr, table->start, table->stop - 1); + /* Move-to-front to exploit common traces */ + list_move(&table->list, &unwind_tables); break; } } diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index b16c07914b55..065d35de0e01 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -40,6 +40,11 @@ SECTIONS __tagtable_begin = .; *(.taglist.init) __tagtable_end = .; +#ifdef CONFIG_SMP_ON_UP + __smpalt_begin = .; + *(.alt.smp.init) + __smpalt_end = .; +#endif INIT_SETUP(16) @@ -104,8 +109,6 @@ SECTIONS RO_DATA(PAGE_SIZE) - _etext = .; /* End of text and rodata section */ - #ifdef CONFIG_ARM_UNWIND /* * Stack unwinding tables @@ -123,6 +126,8 @@ SECTIONS } #endif + _etext = .; /* End of text and rodata section */ + #ifdef CONFIG_XIP_KERNEL __data_loc = ALIGN(4); /* location in binary */ . = PAGE_OFFSET + TEXT_OFFSET; @@ -237,6 +242,12 @@ SECTIONS /* Default discards */ DISCARDS + +#ifndef CONFIG_SMP_ON_UP + /DISCARD/ : { + *(.alt.smp.init) + } +#endif } /* |