summaryrefslogtreecommitdiffstats
path: root/arch/arm/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/kernel')
-rw-r--r--arch/arm/kernel/Makefile1
-rw-r--r--arch/arm/kernel/armksyms.c2
-rw-r--r--arch/arm/kernel/entry-armv.S11
-rw-r--r--arch/arm/kernel/entry-common.S65
-rw-r--r--arch/arm/kernel/etm.c15
-rw-r--r--arch/arm/kernel/ftrace.c188
-rw-r--r--arch/arm/kernel/head-common.S2
-rw-r--r--arch/arm/kernel/head.S50
-rw-r--r--arch/arm/kernel/hw_breakpoint.c849
-rw-r--r--arch/arm/kernel/module.c68
-rw-r--r--arch/arm/kernel/process.c24
-rw-r--r--arch/arm/kernel/ptrace.c239
-rw-r--r--arch/arm/kernel/setup.c6
-rw-r--r--arch/arm/kernel/smp.c3
-rw-r--r--arch/arm/kernel/unwind.c2
-rw-r--r--arch/arm/kernel/vmlinux.lds.S15
16 files changed, 1435 insertions, 105 deletions
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 980b78e31328..5b9b268f4fbb 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -42,6 +42,7 @@ obj-$(CONFIG_KGDB) += kgdb.o
obj-$(CONFIG_ARM_UNWIND) += unwind.o
obj-$(CONFIG_HAVE_TCM) += tcm.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
+obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
obj-$(CONFIG_CRUNCH) += crunch.o crunch-bits.o
AFLAGS_crunch-bits.o := -Wa,-mcpu=ep9312
diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c
index 8214bfebfaca..e5e1e5387678 100644
--- a/arch/arm/kernel/armksyms.c
+++ b/arch/arm/kernel/armksyms.c
@@ -165,6 +165,8 @@ EXPORT_SYMBOL(_find_next_bit_be);
#endif
#ifdef CONFIG_FUNCTION_TRACER
+#ifdef CONFIG_OLD_MCOUNT
EXPORT_SYMBOL(mcount);
+#endif
EXPORT_SYMBOL(__gnu_mcount_nc);
#endif
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index bb8e93a76407..c09e3573c5de 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -46,7 +46,8 @@
* this macro assumes that irqstat (r6) and base (r5) are
* preserved from get_irqnr_and_base above
*/
- test_for_ipi r0, r6, r5, lr
+ ALT_SMP(test_for_ipi r0, r6, r5, lr)
+ ALT_UP_B(9997f)
movne r0, sp
adrne lr, BSYM(1b)
bne do_IPI
@@ -57,6 +58,7 @@
adrne lr, BSYM(1b)
bne do_local_timer
#endif
+9997:
#endif
.endm
@@ -965,11 +967,8 @@ kuser_cmpxchg_fixup:
beq 1b
rsbs r0, r3, #0
/* beware -- each __kuser slot must be 8 instructions max */
-#ifdef CONFIG_SMP
- b __kuser_memory_barrier
-#else
- usr_ret lr
-#endif
+ ALT_SMP(b __kuser_memory_barrier)
+ ALT_UP(usr_ret lr)
#endif
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 1b560825e1cf..2d23ad985180 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -48,6 +48,8 @@ work_pending:
beq no_work_pending
mov r0, sp @ 'regs'
mov r2, why @ 'syscall'
+ tst r1, #_TIF_SIGPENDING @ delivering a signal?
+ movne why, #0 @ prevent further restarts
bl do_notify_resume
b ret_slow_syscall @ Check work again
@@ -127,30 +129,58 @@ ENDPROC(ret_from_fork)
* clobber the ip register. This is OK because the ARM calling convention
* allows it to be clobbered in subroutines and doesn't use it to hold
* parameters.)
+ *
+ * When using dynamic ftrace, we patch out the mcount call by a "mov r0, r0"
+ * for the mcount case, and a "pop {lr}" for the __gnu_mcount_nc case (see
+ * arch/arm/kernel/ftrace.c).
*/
+
+#ifndef CONFIG_OLD_MCOUNT
+#if (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 4))
+#error Ftrace requires CONFIG_FRAME_POINTER=y with GCC older than 4.4.0.
+#endif
+#endif
+
#ifdef CONFIG_DYNAMIC_FTRACE
-ENTRY(mcount)
+ENTRY(__gnu_mcount_nc)
+ mov ip, lr
+ ldmia sp!, {lr}
+ mov pc, ip
+ENDPROC(__gnu_mcount_nc)
+
+ENTRY(ftrace_caller)
stmdb sp!, {r0-r3, lr}
mov r0, lr
sub r0, r0, #MCOUNT_INSN_SIZE
+ ldr r1, [sp, #20]
- .globl mcount_call
-mcount_call:
+ .global ftrace_call
+ftrace_call:
bl ftrace_stub
- ldr lr, [fp, #-4] @ restore lr
- ldmia sp!, {r0-r3, pc}
+ ldmia sp!, {r0-r3, ip, lr}
+ mov pc, ip
+ENDPROC(ftrace_caller)
-ENTRY(ftrace_caller)
+#ifdef CONFIG_OLD_MCOUNT
+ENTRY(mcount)
+ stmdb sp!, {lr}
+ ldr lr, [fp, #-4]
+ ldmia sp!, {pc}
+ENDPROC(mcount)
+
+ENTRY(ftrace_caller_old)
stmdb sp!, {r0-r3, lr}
ldr r1, [fp, #-4]
mov r0, lr
sub r0, r0, #MCOUNT_INSN_SIZE
- .globl ftrace_call
-ftrace_call:
+ .globl ftrace_call_old
+ftrace_call_old:
bl ftrace_stub
ldr lr, [fp, #-4] @ restore lr
ldmia sp!, {r0-r3, pc}
+ENDPROC(ftrace_caller_old)
+#endif
#else
@@ -158,7 +188,7 @@ ENTRY(__gnu_mcount_nc)
stmdb sp!, {r0-r3, lr}
ldr r0, =ftrace_trace_function
ldr r2, [r0]
- adr r0, ftrace_stub
+ adr r0, .Lftrace_stub
cmp r0, r2
bne gnu_trace
ldmia sp!, {r0-r3, ip, lr}
@@ -168,11 +198,19 @@ gnu_trace:
ldr r1, [sp, #20] @ lr of instrumented routine
mov r0, lr
sub r0, r0, #MCOUNT_INSN_SIZE
- mov lr, pc
+ adr lr, BSYM(1f)
mov pc, r2
+1:
ldmia sp!, {r0-r3, ip, lr}
mov pc, ip
+ENDPROC(__gnu_mcount_nc)
+#ifdef CONFIG_OLD_MCOUNT
+/*
+ * This is under an ifdef in order to force link-time errors for people trying
+ * to build with !FRAME_POINTER with a GCC which doesn't use the new-style
+ * mcount.
+ */
ENTRY(mcount)
stmdb sp!, {r0-r3, lr}
ldr r0, =ftrace_trace_function
@@ -191,12 +229,15 @@ trace:
mov pc, r2
ldr lr, [fp, #-4] @ restore lr
ldmia sp!, {r0-r3, pc}
+ENDPROC(mcount)
+#endif
#endif /* CONFIG_DYNAMIC_FTRACE */
- .globl ftrace_stub
-ftrace_stub:
+ENTRY(ftrace_stub)
+.Lftrace_stub:
mov pc, lr
+ENDPROC(ftrace_stub)
#endif /* CONFIG_FUNCTION_TRACER */
diff --git a/arch/arm/kernel/etm.c b/arch/arm/kernel/etm.c
index 33c7077174db..a48d51257988 100644
--- a/arch/arm/kernel/etm.c
+++ b/arch/arm/kernel/etm.c
@@ -30,6 +30,21 @@
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Alexander Shishkin");
+/*
+ * ETM tracer state
+ */
+struct tracectx {
+ unsigned int etb_bufsz;
+ void __iomem *etb_regs;
+ void __iomem *etm_regs;
+ unsigned long flags;
+ int ncmppairs;
+ int etm_portsz;
+ struct device *dev;
+ struct clk *emu_clk;
+ struct mutex mutex;
+};
+
static struct tracectx tracer;
static inline bool trace_isrunning(struct tracectx *t)
diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c
index 0298286ad4ad..971ac8c36ea7 100644
--- a/arch/arm/kernel/ftrace.c
+++ b/arch/arm/kernel/ftrace.c
@@ -2,102 +2,194 @@
* Dynamic function tracing support.
*
* Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com>
+ * Copyright (C) 2010 Rabin Vincent <rabin@rab.in>
*
* For licencing details, see COPYING.
*
* Defines low-level handling of mcount calls when the kernel
* is compiled with the -pg flag. When using dynamic ftrace, the
- * mcount call-sites get patched lazily with NOP till they are
- * enabled. All code mutation routines here take effect atomically.
+ * mcount call-sites get patched with NOP till they are enabled.
+ * All code mutation routines here are called under stop_machine().
*/
#include <linux/ftrace.h>
+#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/ftrace.h>
-#define PC_OFFSET 8
-#define BL_OPCODE 0xeb000000
-#define BL_OFFSET_MASK 0x00ffffff
+#ifdef CONFIG_THUMB2_KERNEL
+#define NOP 0xeb04f85d /* pop.w {lr} */
+#else
+#define NOP 0xe8bd4000 /* pop {lr} */
+#endif
-static unsigned long bl_insn;
-static const unsigned long NOP = 0xe1a00000; /* mov r0, r0 */
+#ifdef CONFIG_OLD_MCOUNT
+#define OLD_MCOUNT_ADDR ((unsigned long) mcount)
+#define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old)
-unsigned char *ftrace_nop_replace(void)
+#define OLD_NOP 0xe1a00000 /* mov r0, r0 */
+
+static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
{
- return (char *)&NOP;
+ return rec->arch.old_mcount ? OLD_NOP : NOP;
}
+static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
+{
+ if (!rec->arch.old_mcount)
+ return addr;
+
+ if (addr == MCOUNT_ADDR)
+ addr = OLD_MCOUNT_ADDR;
+ else if (addr == FTRACE_ADDR)
+ addr = OLD_FTRACE_ADDR;
+
+ return addr;
+}
+#else
+static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
+{
+ return NOP;
+}
+
+static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
+{
+ return addr;
+}
+#endif
+
/* construct a branch (BL) instruction to addr */
-unsigned char *ftrace_call_replace(unsigned long pc, unsigned long addr)
+#ifdef CONFIG_THUMB2_KERNEL
+static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr)
{
+ unsigned long s, j1, j2, i1, i2, imm10, imm11;
+ unsigned long first, second;
long offset;
- offset = (long)addr - (long)(pc + PC_OFFSET);
+ offset = (long)addr - (long)(pc + 4);
+ if (offset < -16777216 || offset > 16777214) {
+ WARN_ON_ONCE(1);
+ return 0;
+ }
+
+ s = (offset >> 24) & 0x1;
+ i1 = (offset >> 23) & 0x1;
+ i2 = (offset >> 22) & 0x1;
+ imm10 = (offset >> 12) & 0x3ff;
+ imm11 = (offset >> 1) & 0x7ff;
+
+ j1 = (!i1) ^ s;
+ j2 = (!i2) ^ s;
+
+ first = 0xf000 | (s << 10) | imm10;
+ second = 0xd000 | (j1 << 13) | (j2 << 11) | imm11;
+
+ return (second << 16) | first;
+}
+#else
+static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr)
+{
+ long offset;
+
+ offset = (long)addr - (long)(pc + 8);
if (unlikely(offset < -33554432 || offset > 33554428)) {
/* Can't generate branches that far (from ARM ARM). Ftrace
* doesn't generate branches outside of kernel text.
*/
WARN_ON_ONCE(1);
- return NULL;
+ return 0;
}
- offset = (offset >> 2) & BL_OFFSET_MASK;
- bl_insn = BL_OPCODE | offset;
- return (unsigned char *)&bl_insn;
-}
-int ftrace_modify_code(unsigned long pc, unsigned char *old_code,
- unsigned char *new_code)
-{
- unsigned long err = 0, replaced = 0, old, new;
+ offset = (offset >> 2) & 0x00ffffff;
- old = *(unsigned long *)old_code;
- new = *(unsigned long *)new_code;
+ return 0xeb000000 | offset;
+}
+#endif
- __asm__ __volatile__ (
- "1: ldr %1, [%2] \n"
- " cmp %1, %4 \n"
- "2: streq %3, [%2] \n"
- " cmpne %1, %3 \n"
- " movne %0, #2 \n"
- "3:\n"
+static int ftrace_modify_code(unsigned long pc, unsigned long old,
+ unsigned long new)
+{
+ unsigned long replaced;
- ".pushsection .fixup, \"ax\"\n"
- "4: mov %0, #1 \n"
- " b 3b \n"
- ".popsection\n"
+ if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE))
+ return -EFAULT;
- ".pushsection __ex_table, \"a\"\n"
- " .long 1b, 4b \n"
- " .long 2b, 4b \n"
- ".popsection\n"
+ if (replaced != old)
+ return -EINVAL;
- : "=r"(err), "=r"(replaced)
- : "r"(pc), "r"(new), "r"(old), "0"(err), "1"(replaced)
- : "memory");
+ if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE))
+ return -EPERM;
- if (!err && (replaced == old))
- flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
+ flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
- return err;
+ return 0;
}
int ftrace_update_ftrace_func(ftrace_func_t func)
{
- int ret;
unsigned long pc, old;
- unsigned char *new;
+ unsigned long new;
+ int ret;
pc = (unsigned long)&ftrace_call;
memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE);
new = ftrace_call_replace(pc, (unsigned long)func);
- ret = ftrace_modify_code(pc, (unsigned char *)&old, new);
+
+ ret = ftrace_modify_code(pc, old, new);
+
+#ifdef CONFIG_OLD_MCOUNT
+ if (!ret) {
+ pc = (unsigned long)&ftrace_call_old;
+ memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE);
+ new = ftrace_call_replace(pc, (unsigned long)func);
+
+ ret = ftrace_modify_code(pc, old, new);
+ }
+#endif
+
+ return ret;
+}
+
+int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
+{
+ unsigned long new, old;
+ unsigned long ip = rec->ip;
+
+ old = ftrace_nop_replace(rec);
+ new = ftrace_call_replace(ip, adjust_address(rec, addr));
+
+ return ftrace_modify_code(rec->ip, old, new);
+}
+
+int ftrace_make_nop(struct module *mod,
+ struct dyn_ftrace *rec, unsigned long addr)
+{
+ unsigned long ip = rec->ip;
+ unsigned long old;
+ unsigned long new;
+ int ret;
+
+ old = ftrace_call_replace(ip, adjust_address(rec, addr));
+ new = ftrace_nop_replace(rec);
+ ret = ftrace_modify_code(ip, old, new);
+
+#ifdef CONFIG_OLD_MCOUNT
+ if (ret == -EINVAL && addr == MCOUNT_ADDR) {
+ rec->arch.old_mcount = true;
+
+ old = ftrace_call_replace(ip, adjust_address(rec, addr));
+ new = ftrace_nop_replace(rec);
+ ret = ftrace_modify_code(ip, old, new);
+ }
+#endif
+
return ret;
}
-/* run from ftrace_init with irqs disabled */
int __init ftrace_dyn_arch_init(void *data)
{
- ftrace_mcount_set(data);
+ *(unsigned long *)data = 0;
+
return 0;
}
diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S
index b9505aa267c0..58a3e632b6d5 100644
--- a/arch/arm/kernel/head-common.S
+++ b/arch/arm/kernel/head-common.S
@@ -20,7 +20,7 @@
__switch_data:
.long __mmap_switched
.long __data_loc @ r4
- .long _data @ r5
+ .long _sdata @ r5
.long __bss_start @ r6
.long _end @ r7
.long processor_id @ r4
diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index eb62bf947212..b44d21e1e344 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -86,6 +86,9 @@ ENTRY(stext)
movs r8, r5 @ invalid machine (r5=0)?
beq __error_a @ yes, error 'a'
bl __vet_atags
+#ifdef CONFIG_SMP_ON_UP
+ bl __fixup_smp
+#endif
bl __create_page_tables
/*
@@ -333,4 +336,51 @@ __create_page_tables:
ENDPROC(__create_page_tables)
.ltorg
+#ifdef CONFIG_SMP_ON_UP
+__fixup_smp:
+ mov r7, #0x00070000
+ orr r6, r7, #0xff000000 @ mask 0xff070000
+ orr r7, r7, #0x41000000 @ val 0x41070000
+ and r0, r9, r6
+ teq r0, r7 @ ARM CPU and ARMv6/v7?
+ bne __fixup_smp_on_up @ no, assume UP
+
+ orr r6, r6, #0x0000ff00
+ orr r6, r6, #0x000000f0 @ mask 0xff07fff0
+ orr r7, r7, #0x0000b000
+ orr r7, r7, #0x00000020 @ val 0x4107b020
+ and r0, r9, r6
+ teq r0, r7 @ ARM 11MPCore?
+ moveq pc, lr @ yes, assume SMP
+
+ mrc p15, 0, r0, c0, c0, 5 @ read MPIDR
+ tst r0, #1 << 31
+ movne pc, lr @ bit 31 => SMP
+
+__fixup_smp_on_up:
+ adr r0, 1f
+ ldmia r0, {r3, r6, r7}
+ sub r3, r0, r3
+ add r6, r6, r3
+ add r7, r7, r3
+2: cmp r6, r7
+ ldmia r6!, {r0, r4}
+ strlo r4, [r0, r3]
+ blo 2b
+ mov pc, lr
+ENDPROC(__fixup_smp)
+
+1: .word .
+ .word __smpalt_begin
+ .word __smpalt_end
+
+ .pushsection .data
+ .globl smp_on_up
+smp_on_up:
+ ALT_SMP(.long 1)
+ ALT_UP(.long 0)
+ .popsection
+
+#endif
+
#include "head-common.S"
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
new file mode 100644
index 000000000000..54593b0c241b
--- /dev/null
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -0,0 +1,849 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) 2009, 2010 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+/*
+ * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
+ * using the CPU's debug registers.
+ */
+#define pr_fmt(fmt) "hw-breakpoint: " fmt
+
+#include <linux/errno.h>
+#include <linux/perf_event.h>
+#include <linux/hw_breakpoint.h>
+#include <linux/smp.h>
+
+#include <asm/cacheflush.h>
+#include <asm/cputype.h>
+#include <asm/current.h>
+#include <asm/hw_breakpoint.h>
+#include <asm/kdebug.h>
+#include <asm/system.h>
+#include <asm/traps.h>
+
+/* Breakpoint currently in use for each BRP. */
+static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
+
+/* Watchpoint currently in use for each WRP. */
+static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
+
+/* Number of BRP/WRP registers on this CPU. */
+static int core_num_brps;
+static int core_num_wrps;
+
+/* Debug architecture version. */
+static u8 debug_arch;
+
+/* Maximum supported watchpoint length. */
+static u8 max_watchpoint_len;
+
+/* Determine number of BRP registers available. */
+static int get_num_brps(void)
+{
+ u32 didr;
+ ARM_DBG_READ(c0, 0, didr);
+ return ((didr >> 24) & 0xf) + 1;
+}
+
+/* Determine number of WRP registers available. */
+static int get_num_wrps(void)
+{
+ /*
+ * FIXME: When a watchpoint fires, the only way to work out which
+ * watchpoint it was is by disassembling the faulting instruction
+ * and working out the address of the memory access.
+ *
+ * Furthermore, we can only do this if the watchpoint was precise
+ * since imprecise watchpoints prevent us from calculating register
+ * based addresses.
+ *
+ * For the time being, we only report 1 watchpoint register so we
+ * always know which watchpoint fired. In the future we can either
+ * add a disassembler and address generation emulator, or we can
+ * insert a check to see if the DFAR is set on watchpoint exception
+ * entry [the ARM ARM states that the DFAR is UNKNOWN, but
+ * experience shows that it is set on some implementations].
+ */
+
+#if 0
+ u32 didr, wrps;
+ ARM_DBG_READ(c0, 0, didr);
+ return ((didr >> 28) & 0xf) + 1;
+#endif
+
+ return 1;
+}
+
+int hw_breakpoint_slots(int type)
+{
+ /*
+ * We can be called early, so don't rely on
+ * our static variables being initialised.
+ */
+ switch (type) {
+ case TYPE_INST:
+ return get_num_brps();
+ case TYPE_DATA:
+ return get_num_wrps();
+ default:
+ pr_warning("unknown slot type: %d\n", type);
+ return 0;
+ }
+}
+
+/* Determine debug architecture. */
+static u8 get_debug_arch(void)
+{
+ u32 didr;
+
+ /* Do we implement the extended CPUID interface? */
+ if (((read_cpuid_id() >> 16) & 0xf) != 0xf) {
+ pr_warning("CPUID feature registers not supported. "
+ "Assuming v6 debug is present.\n");
+ return ARM_DEBUG_ARCH_V6;
+ }
+
+ ARM_DBG_READ(c0, 0, didr);
+ return (didr >> 16) & 0xf;
+}
+
+/* Does this core support mismatch breakpoints? */
+static int core_has_mismatch_bps(void)
+{
+ return debug_arch >= ARM_DEBUG_ARCH_V7_ECP14 && core_num_brps > 1;
+}
+
+u8 arch_get_debug_arch(void)
+{
+ return debug_arch;
+}
+
+#define READ_WB_REG_CASE(OP2, M, VAL) \
+ case ((OP2 << 4) + M): \
+ ARM_DBG_READ(c ## M, OP2, VAL); \
+ break
+
+#define WRITE_WB_REG_CASE(OP2, M, VAL) \
+ case ((OP2 << 4) + M): \
+ ARM_DBG_WRITE(c ## M, OP2, VAL);\
+ break
+
+#define GEN_READ_WB_REG_CASES(OP2, VAL) \
+ READ_WB_REG_CASE(OP2, 0, VAL); \
+ READ_WB_REG_CASE(OP2, 1, VAL); \
+ READ_WB_REG_CASE(OP2, 2, VAL); \
+ READ_WB_REG_CASE(OP2, 3, VAL); \
+ READ_WB_REG_CASE(OP2, 4, VAL); \
+ READ_WB_REG_CASE(OP2, 5, VAL); \
+ READ_WB_REG_CASE(OP2, 6, VAL); \
+ READ_WB_REG_CASE(OP2, 7, VAL); \
+ READ_WB_REG_CASE(OP2, 8, VAL); \
+ READ_WB_REG_CASE(OP2, 9, VAL); \
+ READ_WB_REG_CASE(OP2, 10, VAL); \
+ READ_WB_REG_CASE(OP2, 11, VAL); \
+ READ_WB_REG_CASE(OP2, 12, VAL); \
+ READ_WB_REG_CASE(OP2, 13, VAL); \
+ READ_WB_REG_CASE(OP2, 14, VAL); \
+ READ_WB_REG_CASE(OP2, 15, VAL)
+
+#define GEN_WRITE_WB_REG_CASES(OP2, VAL) \
+ WRITE_WB_REG_CASE(OP2, 0, VAL); \
+ WRITE_WB_REG_CASE(OP2, 1, VAL); \
+ WRITE_WB_REG_CASE(OP2, 2, VAL); \
+ WRITE_WB_REG_CASE(OP2, 3, VAL); \
+ WRITE_WB_REG_CASE(OP2, 4, VAL); \
+ WRITE_WB_REG_CASE(OP2, 5, VAL); \
+ WRITE_WB_REG_CASE(OP2, 6, VAL); \
+ WRITE_WB_REG_CASE(OP2, 7, VAL); \
+ WRITE_WB_REG_CASE(OP2, 8, VAL); \
+ WRITE_WB_REG_CASE(OP2, 9, VAL); \
+ WRITE_WB_REG_CASE(OP2, 10, VAL); \
+ WRITE_WB_REG_CASE(OP2, 11, VAL); \
+ WRITE_WB_REG_CASE(OP2, 12, VAL); \
+ WRITE_WB_REG_CASE(OP2, 13, VAL); \
+ WRITE_WB_REG_CASE(OP2, 14, VAL); \
+ WRITE_WB_REG_CASE(OP2, 15, VAL)
+
+static u32 read_wb_reg(int n)
+{
+ u32 val = 0;
+
+ switch (n) {
+ GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
+ GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
+ GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
+ GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
+ default:
+ pr_warning("attempt to read from unknown breakpoint "
+ "register %d\n", n);
+ }
+
+ return val;
+}
+
+static void write_wb_reg(int n, u32 val)
+{
+ switch (n) {
+ GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
+ GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
+ GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
+ GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
+ default:
+ pr_warning("attempt to write to unknown breakpoint "
+ "register %d\n", n);
+ }
+ isb();
+}
+
+/*
+ * In order to access the breakpoint/watchpoint control registers,
+ * we must be running in debug monitor mode. Unfortunately, we can
+ * be put into halting debug mode at any time by an external debugger
+ * but there is nothing we can do to prevent that.
+ */
+static int enable_monitor_mode(void)
+{
+ u32 dscr;
+ int ret = 0;
+
+ ARM_DBG_READ(c1, 0, dscr);
+
+ /* Ensure that halting mode is disabled. */
+ if (WARN_ONCE(dscr & ARM_DSCR_HDBGEN, "halting debug mode enabled."
+ "Unable to access hardware resources.")) {
+ ret = -EPERM;
+ goto out;
+ }
+
+ /* Write to the corresponding DSCR. */
+ switch (debug_arch) {
+ case ARM_DEBUG_ARCH_V6:
+ case ARM_DEBUG_ARCH_V6_1:
+ ARM_DBG_WRITE(c1, 0, (dscr | ARM_DSCR_MDBGEN));
+ break;
+ case ARM_DEBUG_ARCH_V7_ECP14:
+ ARM_DBG_WRITE(c2, 2, (dscr | ARM_DSCR_MDBGEN));
+ break;
+ default:
+ ret = -ENODEV;
+ goto out;
+ }
+
+ /* Check that the write made it through. */
+ ARM_DBG_READ(c1, 0, dscr);
+ if (WARN_ONCE(!(dscr & ARM_DSCR_MDBGEN),
+ "failed to enable monitor mode.")) {
+ ret = -EPERM;
+ }
+
+out:
+ return ret;
+}
+
+/*
+ * Check if 8-bit byte-address select is available.
+ * This clobbers WRP 0.
+ */
+static u8 get_max_wp_len(void)
+{
+ u32 ctrl_reg;
+ struct arch_hw_breakpoint_ctrl ctrl;
+ u8 size = 4;
+
+ if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
+ goto out;
+
+ if (enable_monitor_mode())
+ goto out;
+
+ memset(&ctrl, 0, sizeof(ctrl));
+ ctrl.len = ARM_BREAKPOINT_LEN_8;
+ ctrl_reg = encode_ctrl_reg(ctrl);
+
+ write_wb_reg(ARM_BASE_WVR, 0);
+ write_wb_reg(ARM_BASE_WCR, ctrl_reg);
+ if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
+ size = 8;
+
+out:
+ return size;
+}
+
+u8 arch_get_max_wp_len(void)
+{
+ return max_watchpoint_len;
+}
+
+/*
+ * Handler for reactivating a suspended watchpoint when the single
+ * step `mismatch' breakpoint is triggered.
+ */
+static void wp_single_step_handler(struct perf_event *bp, int unused,
+ struct perf_sample_data *data,
+ struct pt_regs *regs)
+{
+ perf_event_enable(counter_arch_bp(bp)->suspended_wp);
+ unregister_hw_breakpoint(bp);
+}
+
+static int bp_is_single_step(struct perf_event *bp)
+{
+ return bp->overflow_handler == wp_single_step_handler;
+}
+
+/*
+ * Install a perf counter breakpoint.
+ */
+int arch_install_hw_breakpoint(struct perf_event *bp)
+{
+ struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+ struct perf_event **slot, **slots;
+ int i, max_slots, ctrl_base, val_base, ret = 0;
+
+ /* Ensure that we are in monitor mode and halting mode is disabled. */
+ ret = enable_monitor_mode();
+ if (ret)
+ goto out;
+
+ if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
+ /* Breakpoint */
+ ctrl_base = ARM_BASE_BCR;
+ val_base = ARM_BASE_BVR;
+ slots = __get_cpu_var(bp_on_reg);
+ max_slots = core_num_brps - 1;
+
+ if (bp_is_single_step(bp)) {
+ info->ctrl.mismatch = 1;
+ i = max_slots;
+ slots[i] = bp;
+ goto setup;
+ }
+ } else {
+ /* Watchpoint */
+ ctrl_base = ARM_BASE_WCR;
+ val_base = ARM_BASE_WVR;
+ slots = __get_cpu_var(wp_on_reg);
+ max_slots = core_num_wrps;
+ }
+
+ for (i = 0; i < max_slots; ++i) {
+ slot = &slots[i];
+
+ if (!*slot) {
+ *slot = bp;
+ break;
+ }
+ }
+
+ if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot")) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+setup:
+ /* Setup the address register. */
+ write_wb_reg(val_base + i, info->address);
+
+ /* Setup the control register. */
+ write_wb_reg(ctrl_base + i, encode_ctrl_reg(info->ctrl) | 0x1);
+
+out:
+ return ret;
+}
+
+void arch_uninstall_hw_breakpoint(struct perf_event *bp)
+{
+ struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+ struct perf_event **slot, **slots;
+ int i, max_slots, base;
+
+ if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
+ /* Breakpoint */
+ base = ARM_BASE_BCR;
+ slots = __get_cpu_var(bp_on_reg);
+ max_slots = core_num_brps - 1;
+
+ if (bp_is_single_step(bp)) {
+ i = max_slots;
+ slots[i] = NULL;
+ goto reset;
+ }
+ } else {
+ /* Watchpoint */
+ base = ARM_BASE_WCR;
+ slots = __get_cpu_var(wp_on_reg);
+ max_slots = core_num_wrps;
+ }
+
+ /* Remove the breakpoint. */
+ for (i = 0; i < max_slots; ++i) {
+ slot = &slots[i];
+
+ if (*slot == bp) {
+ *slot = NULL;
+ break;
+ }
+ }
+
+ if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
+ return;
+
+reset:
+ /* Reset the control register. */
+ write_wb_reg(base + i, 0);
+}
+
+static int get_hbp_len(u8 hbp_len)
+{
+ unsigned int len_in_bytes = 0;
+
+ switch (hbp_len) {
+ case ARM_BREAKPOINT_LEN_1:
+ len_in_bytes = 1;
+ break;
+ case ARM_BREAKPOINT_LEN_2:
+ len_in_bytes = 2;
+ break;
+ case ARM_BREAKPOINT_LEN_4:
+ len_in_bytes = 4;
+ break;
+ case ARM_BREAKPOINT_LEN_8:
+ len_in_bytes = 8;
+ break;
+ }
+
+ return len_in_bytes;
+}
+
+/*
+ * Check whether bp virtual address is in kernel space.
+ */
+int arch_check_bp_in_kernelspace(struct perf_event *bp)
+{
+ unsigned int len;
+ unsigned long va;
+ struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+
+ va = info->address;
+ len = get_hbp_len(info->ctrl.len);
+
+ return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
+}
+
+/*
+ * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
+ * Hopefully this will disappear when ptrace can bypass the conversion
+ * to generic breakpoint descriptions.
+ */
+int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
+ int *gen_len, int *gen_type)
+{
+ /* Type */
+ switch (ctrl.type) {
+ case ARM_BREAKPOINT_EXECUTE:
+ *gen_type = HW_BREAKPOINT_X;
+ break;
+ case ARM_BREAKPOINT_LOAD:
+ *gen_type = HW_BREAKPOINT_R;
+ break;
+ case ARM_BREAKPOINT_STORE:
+ *gen_type = HW_BREAKPOINT_W;
+ break;
+ case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
+ *gen_type = HW_BREAKPOINT_RW;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* Len */
+ switch (ctrl.len) {
+ case ARM_BREAKPOINT_LEN_1:
+ *gen_len = HW_BREAKPOINT_LEN_1;
+ break;
+ case ARM_BREAKPOINT_LEN_2:
+ *gen_len = HW_BREAKPOINT_LEN_2;
+ break;
+ case ARM_BREAKPOINT_LEN_4:
+ *gen_len = HW_BREAKPOINT_LEN_4;
+ break;
+ case ARM_BREAKPOINT_LEN_8:
+ *gen_len = HW_BREAKPOINT_LEN_8;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * Construct an arch_hw_breakpoint from a perf_event.
+ */
+static int arch_build_bp_info(struct perf_event *bp)
+{
+ struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+
+ /* Type */
+ switch (bp->attr.bp_type) {
+ case HW_BREAKPOINT_X:
+ info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
+ break;
+ case HW_BREAKPOINT_R:
+ info->ctrl.type = ARM_BREAKPOINT_LOAD;
+ break;
+ case HW_BREAKPOINT_W:
+ info->ctrl.type = ARM_BREAKPOINT_STORE;
+ break;
+ case HW_BREAKPOINT_RW:
+ info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* Len */
+ switch (bp->attr.bp_len) {
+ case HW_BREAKPOINT_LEN_1:
+ info->ctrl.len = ARM_BREAKPOINT_LEN_1;
+ break;
+ case HW_BREAKPOINT_LEN_2:
+ info->ctrl.len = ARM_BREAKPOINT_LEN_2;
+ break;
+ case HW_BREAKPOINT_LEN_4:
+ info->ctrl.len = ARM_BREAKPOINT_LEN_4;
+ break;
+ case HW_BREAKPOINT_LEN_8:
+ info->ctrl.len = ARM_BREAKPOINT_LEN_8;
+ if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE)
+ && max_watchpoint_len >= 8)
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* Address */
+ info->address = bp->attr.bp_addr;
+
+ /* Privilege */
+ info->ctrl.privilege = ARM_BREAKPOINT_USER;
+ if (arch_check_bp_in_kernelspace(bp) && !bp_is_single_step(bp))
+ info->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
+
+ /* Enabled? */
+ info->ctrl.enabled = !bp->attr.disabled;
+
+ /* Mismatch */
+ info->ctrl.mismatch = 0;
+
+ return 0;
+}
+
+/*
+ * Validate the arch-specific HW Breakpoint register settings.
+ */
+int arch_validate_hwbkpt_settings(struct perf_event *bp)
+{
+ struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+ int ret = 0;
+ u32 bytelen, max_len, offset, alignment_mask = 0x3;
+
+ /* Build the arch_hw_breakpoint. */
+ ret = arch_build_bp_info(bp);
+ if (ret)
+ goto out;
+
+ /* Check address alignment. */
+ if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
+ alignment_mask = 0x7;
+ if (info->address & alignment_mask) {
+ /*
+ * Try to fix the alignment. This may result in a length
+ * that is too large, so we must check for that.
+ */
+ bytelen = get_hbp_len(info->ctrl.len);
+ max_len = info->ctrl.type == ARM_BREAKPOINT_EXECUTE ? 4 :
+ max_watchpoint_len;
+
+ if (max_len >= 8)
+ offset = info->address & 0x7;
+ else
+ offset = info->address & 0x3;
+
+ if (bytelen > (1 << ((max_len - (offset + 1)) >> 1))) {
+ ret = -EFBIG;
+ goto out;
+ }
+
+ info->ctrl.len <<= offset;
+ info->address &= ~offset;
+
+ pr_debug("breakpoint alignment fixup: length = 0x%x, "
+ "address = 0x%x\n", info->ctrl.len, info->address);
+ }
+
+ /*
+ * Currently we rely on an overflow handler to take
+ * care of single-stepping the breakpoint when it fires.
+ * In the case of userspace breakpoints on a core with V7 debug,
+ * we can use the mismatch feature as a poor-man's hardware single-step.
+ */
+ if (WARN_ONCE(!bp->overflow_handler &&
+ (arch_check_bp_in_kernelspace(bp) || !core_has_mismatch_bps()),
+ "overflow handler required but none found")) {
+ ret = -EINVAL;
+ goto out;
+ }
+out:
+ return ret;
+}
+
+static void update_mismatch_flag(int idx, int flag)
+{
+ struct perf_event *bp = __get_cpu_var(bp_on_reg[idx]);
+ struct arch_hw_breakpoint *info;
+
+ if (bp == NULL)
+ return;
+
+ info = counter_arch_bp(bp);
+
+ /* Update the mismatch field to enter/exit `single-step' mode */
+ if (!bp->overflow_handler && info->ctrl.mismatch != flag) {
+ info->ctrl.mismatch = flag;
+ write_wb_reg(ARM_BASE_BCR + idx, encode_ctrl_reg(info->ctrl) | 0x1);
+ }
+}
+
+static void watchpoint_handler(unsigned long unknown, struct pt_regs *regs)
+{
+ int i;
+ struct perf_event *bp, **slots = __get_cpu_var(wp_on_reg);
+ struct arch_hw_breakpoint *info;
+ struct perf_event_attr attr;
+
+ /* Without a disassembler, we can only handle 1 watchpoint. */
+ BUG_ON(core_num_wrps > 1);
+
+ hw_breakpoint_init(&attr);
+ attr.bp_addr = regs->ARM_pc & ~0x3;
+ attr.bp_len = HW_BREAKPOINT_LEN_4;
+ attr.bp_type = HW_BREAKPOINT_X;
+
+ for (i = 0; i < core_num_wrps; ++i) {
+ rcu_read_lock();
+
+ if (slots[i] == NULL) {
+ rcu_read_unlock();
+ continue;
+ }
+
+ /*
+ * The DFAR is an unknown value. Since we only allow a
+ * single watchpoint, we can set the trigger to the lowest
+ * possible faulting address.
+ */
+ info = counter_arch_bp(slots[i]);
+ info->trigger = slots[i]->attr.bp_addr;
+ pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
+ perf_bp_event(slots[i], regs);
+
+ /*
+ * If no overflow handler is present, insert a temporary
+ * mismatch breakpoint so we can single-step over the
+ * watchpoint trigger.
+ */
+ if (!slots[i]->overflow_handler) {
+ bp = register_user_hw_breakpoint(&attr,
+ wp_single_step_handler,
+ current);
+ counter_arch_bp(bp)->suspended_wp = slots[i];
+ perf_event_disable(slots[i]);
+ }
+
+ rcu_read_unlock();
+ }
+}
+
+static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
+{
+ int i;
+ int mismatch;
+ u32 ctrl_reg, val, addr;
+ struct perf_event *bp, **slots = __get_cpu_var(bp_on_reg);
+ struct arch_hw_breakpoint *info;
+ struct arch_hw_breakpoint_ctrl ctrl;
+
+ /* The exception entry code places the amended lr in the PC. */
+ addr = regs->ARM_pc;
+
+ for (i = 0; i < core_num_brps; ++i) {
+ rcu_read_lock();
+
+ bp = slots[i];
+
+ if (bp == NULL) {
+ rcu_read_unlock();
+ continue;
+ }
+
+ mismatch = 0;
+
+ /* Check if the breakpoint value matches. */
+ val = read_wb_reg(ARM_BASE_BVR + i);
+ if (val != (addr & ~0x3))
+ goto unlock;
+
+ /* Possible match, check the byte address select to confirm. */
+ ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
+ decode_ctrl_reg(ctrl_reg, &ctrl);
+ if ((1 << (addr & 0x3)) & ctrl.len) {
+ mismatch = 1;
+ info = counter_arch_bp(bp);
+ info->trigger = addr;
+ }
+
+unlock:
+ if ((mismatch && !info->ctrl.mismatch) || bp_is_single_step(bp)) {
+ pr_debug("breakpoint fired: address = 0x%x\n", addr);
+ perf_bp_event(bp, regs);
+ }
+
+ update_mismatch_flag(i, mismatch);
+ rcu_read_unlock();
+ }
+}
+
+/*
+ * Called from either the Data Abort Handler [watchpoint] or the
+ * Prefetch Abort Handler [breakpoint].
+ */
+static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
+ struct pt_regs *regs)
+{
+ int ret = 1; /* Unhandled fault. */
+ u32 dscr;
+
+ /* We only handle watchpoints and hardware breakpoints. */
+ ARM_DBG_READ(c1, 0, dscr);
+
+ /* Perform perf callbacks. */
+ switch (ARM_DSCR_MOE(dscr)) {
+ case ARM_ENTRY_BREAKPOINT:
+ breakpoint_handler(addr, regs);
+ break;
+ case ARM_ENTRY_ASYNC_WATCHPOINT:
+ WARN_ON("Asynchronous watchpoint exception taken. "
+ "Debugging results may be unreliable");
+ case ARM_ENTRY_SYNC_WATCHPOINT:
+ watchpoint_handler(addr, regs);
+ break;
+ default:
+ goto out;
+ }
+
+ ret = 0;
+out:
+ return ret;
+}
+
+/*
+ * One-time initialisation.
+ */
+static void __init reset_ctrl_regs(void *unused)
+{
+ int i;
+
+ if (enable_monitor_mode())
+ return;
+
+ for (i = 0; i < core_num_brps; ++i) {
+ write_wb_reg(ARM_BASE_BCR + i, 0UL);
+ write_wb_reg(ARM_BASE_BVR + i, 0UL);
+ }
+
+ for (i = 0; i < core_num_wrps; ++i) {
+ write_wb_reg(ARM_BASE_WCR + i, 0UL);
+ write_wb_reg(ARM_BASE_WVR + i, 0UL);
+ }
+}
+
+static int __init arch_hw_breakpoint_init(void)
+{
+ int ret = 0;
+ u32 dscr;
+
+ debug_arch = get_debug_arch();
+
+ if (debug_arch > ARM_DEBUG_ARCH_V7_ECP14) {
+ pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
+ ret = -ENODEV;
+ goto out;
+ }
+
+ /* Determine how many BRPs/WRPs are available. */
+ core_num_brps = get_num_brps();
+ core_num_wrps = get_num_wrps();
+
+ pr_info("found %d breakpoint and %d watchpoint registers.\n",
+ core_num_brps, core_num_wrps);
+
+ if (core_has_mismatch_bps())
+ pr_info("1 breakpoint reserved for watchpoint single-step.\n");
+
+ ARM_DBG_READ(c1, 0, dscr);
+ if (dscr & ARM_DSCR_HDBGEN) {
+ pr_warning("halting debug mode enabled. Assuming maximum "
+ "watchpoint size of 4 bytes.");
+ } else {
+ /* Work out the maximum supported watchpoint length. */
+ max_watchpoint_len = get_max_wp_len();
+ pr_info("maximum watchpoint size is %u bytes.\n",
+ max_watchpoint_len);
+
+ /*
+ * Reset the breakpoint resources. We assume that a halting
+ * debugger will leave the world in a nice state for us.
+ */
+ smp_call_function(reset_ctrl_regs, NULL, 1);
+ reset_ctrl_regs(NULL);
+ }
+
+ /* Register debug fault handler. */
+ hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT,
+ "watchpoint debug exception");
+ hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT,
+ "breakpoint debug exception");
+
+out:
+ return ret;
+}
+arch_initcall(arch_hw_breakpoint_init);
+
+void hw_breakpoint_pmu_read(struct perf_event *bp)
+{
+}
+
+/*
+ * Dummy function to register with die_notifier.
+ */
+int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
+ unsigned long val, void *data)
+{
+ return NOTIFY_DONE;
+}
diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c
index 6b4605893f1e..d9bd786ce23d 100644
--- a/arch/arm/kernel/module.c
+++ b/arch/arm/kernel/module.c
@@ -69,20 +69,31 @@ int module_frob_arch_sections(Elf_Ehdr *hdr,
{
#ifdef CONFIG_ARM_UNWIND
Elf_Shdr *s, *sechdrs_end = sechdrs + hdr->e_shnum;
+ struct arm_unwind_mapping *maps = mod->arch.map;
for (s = sechdrs; s < sechdrs_end; s++) {
- if (strcmp(".ARM.exidx.init.text", secstrings + s->sh_name) == 0)
- mod->arch.unw_sec_init = s;
- else if (strcmp(".ARM.exidx.devinit.text", secstrings + s->sh_name) == 0)
- mod->arch.unw_sec_devinit = s;
- else if (strcmp(".ARM.exidx", secstrings + s->sh_name) == 0)
- mod->arch.unw_sec_core = s;
- else if (strcmp(".init.text", secstrings + s->sh_name) == 0)
- mod->arch.sec_init_text = s;
- else if (strcmp(".devinit.text", secstrings + s->sh_name) == 0)
- mod->arch.sec_devinit_text = s;
- else if (strcmp(".text", secstrings + s->sh_name) == 0)
- mod->arch.sec_core_text = s;
+ char const *secname = secstrings + s->sh_name;
+
+ if (strcmp(".ARM.exidx.init.text", secname) == 0)
+ maps[ARM_SEC_INIT].unw_sec = s;
+ else if (strcmp(".ARM.exidx.devinit.text", secname) == 0)
+ maps[ARM_SEC_DEVINIT].unw_sec = s;
+ else if (strcmp(".ARM.exidx", secname) == 0)
+ maps[ARM_SEC_CORE].unw_sec = s;
+ else if (strcmp(".ARM.exidx.exit.text", secname) == 0)
+ maps[ARM_SEC_EXIT].unw_sec = s;
+ else if (strcmp(".ARM.exidx.devexit.text", secname) == 0)
+ maps[ARM_SEC_DEVEXIT].unw_sec = s;
+ else if (strcmp(".init.text", secname) == 0)
+ maps[ARM_SEC_INIT].sec_text = s;
+ else if (strcmp(".devinit.text", secname) == 0)
+ maps[ARM_SEC_DEVINIT].sec_text = s;
+ else if (strcmp(".text", secname) == 0)
+ maps[ARM_SEC_CORE].sec_text = s;
+ else if (strcmp(".exit.text", secname) == 0)
+ maps[ARM_SEC_EXIT].sec_text = s;
+ else if (strcmp(".devexit.text", secname) == 0)
+ maps[ARM_SEC_DEVEXIT].sec_text = s;
}
#endif
return 0;
@@ -292,31 +303,22 @@ apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab,
#ifdef CONFIG_ARM_UNWIND
static void register_unwind_tables(struct module *mod)
{
- if (mod->arch.unw_sec_init && mod->arch.sec_init_text)
- mod->arch.unwind_init =
- unwind_table_add(mod->arch.unw_sec_init->sh_addr,
- mod->arch.unw_sec_init->sh_size,
- mod->arch.sec_init_text->sh_addr,
- mod->arch.sec_init_text->sh_size);
- if (mod->arch.unw_sec_devinit && mod->arch.sec_devinit_text)
- mod->arch.unwind_devinit =
- unwind_table_add(mod->arch.unw_sec_devinit->sh_addr,
- mod->arch.unw_sec_devinit->sh_size,
- mod->arch.sec_devinit_text->sh_addr,
- mod->arch.sec_devinit_text->sh_size);
- if (mod->arch.unw_sec_core && mod->arch.sec_core_text)
- mod->arch.unwind_core =
- unwind_table_add(mod->arch.unw_sec_core->sh_addr,
- mod->arch.unw_sec_core->sh_size,
- mod->arch.sec_core_text->sh_addr,
- mod->arch.sec_core_text->sh_size);
+ int i;
+ for (i = 0; i < ARM_SEC_MAX; ++i) {
+ struct arm_unwind_mapping *map = &mod->arch.map[i];
+ if (map->unw_sec && map->sec_text)
+ map->unwind = unwind_table_add(map->unw_sec->sh_addr,
+ map->unw_sec->sh_size,
+ map->sec_text->sh_addr,
+ map->sec_text->sh_size);
+ }
}
static void unregister_unwind_tables(struct module *mod)
{
- unwind_table_del(mod->arch.unwind_init);
- unwind_table_del(mod->arch.unwind_devinit);
- unwind_table_del(mod->arch.unwind_core);
+ int i = ARM_SEC_MAX;
+ while (--i >= 0)
+ unwind_table_del(mod->arch.map[i].unwind);
}
#else
static inline void register_unwind_tables(struct module *mod) { }
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 401e38be1f78..3af34bf4f4df 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -29,6 +29,7 @@
#include <linux/utsname.h>
#include <linux/uaccess.h>
#include <linux/random.h>
+#include <linux/hw_breakpoint.h>
#include <asm/cacheflush.h>
#include <asm/leds.h>
@@ -135,6 +136,25 @@ EXPORT_SYMBOL(pm_power_off);
void (*arm_pm_restart)(char str, const char *cmd) = arm_machine_restart;
EXPORT_SYMBOL_GPL(arm_pm_restart);
+static void do_nothing(void *unused)
+{
+}
+
+/*
+ * cpu_idle_wait - Used to ensure that all the CPUs discard old value of
+ * pm_idle and update to new pm_idle value. Required while changing pm_idle
+ * handler on SMP systems.
+ *
+ * Caller must have changed pm_idle to the new value before the call. Old
+ * pm_idle value will not be used by any CPU after the return of this function.
+ */
+void cpu_idle_wait(void)
+{
+ smp_mb();
+ /* kick all the CPUs so that they exit out of pm_idle */
+ smp_call_function(do_nothing, NULL, 1);
+}
+EXPORT_SYMBOL_GPL(cpu_idle_wait);
/*
* This is our default idle handler. We need to disable
@@ -317,6 +337,8 @@ void flush_thread(void)
struct thread_info *thread = current_thread_info();
struct task_struct *tsk = current;
+ flush_ptrace_hw_breakpoint(tsk);
+
memset(thread->used_cp, 0, sizeof(thread->used_cp));
memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
memset(&thread->fpstate, 0, sizeof(union fp_state));
@@ -345,6 +367,8 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start,
thread->cpu_context.sp = (unsigned long)childregs;
thread->cpu_context.pc = (unsigned long)ret_from_fork;
+ clear_ptrace_hw_breakpoint(p);
+
if (clone_flags & CLONE_SETTLS)
thread->tp_value = regs->ARM_r3;
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index f99d489822d5..e0cb6370ed14 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -19,6 +19,8 @@
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/uaccess.h>
+#include <linux/perf_event.h>
+#include <linux/hw_breakpoint.h>
#include <asm/pgtable.h>
#include <asm/system.h>
@@ -847,6 +849,232 @@ static int ptrace_setvfpregs(struct task_struct *tsk, void __user *data)
}
#endif
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+/*
+ * Convert a virtual register number into an index for a thread_info
+ * breakpoint array. Breakpoints are identified using positive numbers
+ * whilst watchpoints are negative. The registers are laid out as pairs
+ * of (address, control), each pair mapping to a unique hw_breakpoint struct.
+ * Register 0 is reserved for describing resource information.
+ */
+static int ptrace_hbp_num_to_idx(long num)
+{
+ if (num < 0)
+ num = (ARM_MAX_BRP << 1) - num;
+ return (num - 1) >> 1;
+}
+
+/*
+ * Returns the virtual register number for the address of the
+ * breakpoint at index idx.
+ */
+static long ptrace_hbp_idx_to_num(int idx)
+{
+ long mid = ARM_MAX_BRP << 1;
+ long num = (idx << 1) + 1;
+ return num > mid ? mid - num : num;
+}
+
+/*
+ * Handle hitting a HW-breakpoint.
+ */
+static void ptrace_hbptriggered(struct perf_event *bp, int unused,
+ struct perf_sample_data *data,
+ struct pt_regs *regs)
+{
+ struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
+ long num;
+ int i;
+ siginfo_t info;
+
+ for (i = 0; i < ARM_MAX_HBP_SLOTS; ++i)
+ if (current->thread.debug.hbp[i] == bp)
+ break;
+
+ num = (i == ARM_MAX_HBP_SLOTS) ? 0 : ptrace_hbp_idx_to_num(i);
+
+ info.si_signo = SIGTRAP;
+ info.si_errno = (int)num;
+ info.si_code = TRAP_HWBKPT;
+ info.si_addr = (void __user *)(bkpt->trigger);
+
+ force_sig_info(SIGTRAP, &info, current);
+}
+
+/*
+ * Set ptrace breakpoint pointers to zero for this task.
+ * This is required in order to prevent child processes from unregistering
+ * breakpoints held by their parent.
+ */
+void clear_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+ memset(tsk->thread.debug.hbp, 0, sizeof(tsk->thread.debug.hbp));
+}
+
+/*
+ * Unregister breakpoints from this task and reset the pointers in
+ * the thread_struct.
+ */
+void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+ int i;
+ struct thread_struct *t = &tsk->thread;
+
+ for (i = 0; i < ARM_MAX_HBP_SLOTS; i++) {
+ if (t->debug.hbp[i]) {
+ unregister_hw_breakpoint(t->debug.hbp[i]);
+ t->debug.hbp[i] = NULL;
+ }
+ }
+}
+
+static u32 ptrace_get_hbp_resource_info(void)
+{
+ u8 num_brps, num_wrps, debug_arch, wp_len;
+ u32 reg = 0;
+
+ num_brps = hw_breakpoint_slots(TYPE_INST);
+ num_wrps = hw_breakpoint_slots(TYPE_DATA);
+ debug_arch = arch_get_debug_arch();
+ wp_len = arch_get_max_wp_len();
+
+ reg |= debug_arch;
+ reg <<= 8;
+ reg |= wp_len;
+ reg <<= 8;
+ reg |= num_wrps;
+ reg <<= 8;
+ reg |= num_brps;
+
+ return reg;
+}
+
+static struct perf_event *ptrace_hbp_create(struct task_struct *tsk, int type)
+{
+ struct perf_event_attr attr;
+
+ ptrace_breakpoint_init(&attr);
+
+ /* Initialise fields to sane defaults. */
+ attr.bp_addr = 0;
+ attr.bp_len = HW_BREAKPOINT_LEN_4;
+ attr.bp_type = type;
+ attr.disabled = 1;
+
+ return register_user_hw_breakpoint(&attr, ptrace_hbptriggered, tsk);
+}
+
+static int ptrace_gethbpregs(struct task_struct *tsk, long num,
+ unsigned long __user *data)
+{
+ u32 reg;
+ int idx, ret = 0;
+ struct perf_event *bp;
+ struct arch_hw_breakpoint_ctrl arch_ctrl;
+
+ if (num == 0) {
+ reg = ptrace_get_hbp_resource_info();
+ } else {
+ idx = ptrace_hbp_num_to_idx(num);
+ if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ bp = tsk->thread.debug.hbp[idx];
+ if (!bp) {
+ reg = 0;
+ goto put;
+ }
+
+ arch_ctrl = counter_arch_bp(bp)->ctrl;
+
+ /*
+ * Fix up the len because we may have adjusted it
+ * to compensate for an unaligned address.
+ */
+ while (!(arch_ctrl.len & 0x1))
+ arch_ctrl.len >>= 1;
+
+ if (idx & 0x1)
+ reg = encode_ctrl_reg(arch_ctrl);
+ else
+ reg = bp->attr.bp_addr;
+ }
+
+put:
+ if (put_user(reg, data))
+ ret = -EFAULT;
+
+out:
+ return ret;
+}
+
+static int ptrace_sethbpregs(struct task_struct *tsk, long num,
+ unsigned long __user *data)
+{
+ int idx, gen_len, gen_type, implied_type, ret = 0;
+ u32 user_val;
+ struct perf_event *bp;
+ struct arch_hw_breakpoint_ctrl ctrl;
+ struct perf_event_attr attr;
+
+ if (num == 0)
+ goto out;
+ else if (num < 0)
+ implied_type = HW_BREAKPOINT_RW;
+ else
+ implied_type = HW_BREAKPOINT_X;
+
+ idx = ptrace_hbp_num_to_idx(num);
+ if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (get_user(user_val, data)) {
+ ret = -EFAULT;
+ goto out;
+ }
+
+ bp = tsk->thread.debug.hbp[idx];
+ if (!bp) {
+ bp = ptrace_hbp_create(tsk, implied_type);
+ if (IS_ERR(bp)) {
+ ret = PTR_ERR(bp);
+ goto out;
+ }
+ tsk->thread.debug.hbp[idx] = bp;
+ }
+
+ attr = bp->attr;
+
+ if (num & 0x1) {
+ /* Address */
+ attr.bp_addr = user_val;
+ } else {
+ /* Control */
+ decode_ctrl_reg(user_val, &ctrl);
+ ret = arch_bp_generic_fields(ctrl, &gen_len, &gen_type);
+ if (ret)
+ goto out;
+
+ if ((gen_type & implied_type) != gen_type) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ attr.bp_len = gen_len;
+ attr.bp_type = gen_type;
+ attr.disabled = !ctrl.enabled;
+ }
+
+ ret = modify_user_hw_breakpoint(bp, &attr);
+out:
+ return ret;
+}
+#endif
+
long arch_ptrace(struct task_struct *child, long request, long addr, long data)
{
int ret;
@@ -916,6 +1144,17 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data)
break;
#endif
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+ case PTRACE_GETHBPREGS:
+ ret = ptrace_gethbpregs(child, addr,
+ (unsigned long __user *)data);
+ break;
+ case PTRACE_SETHBPREGS:
+ ret = ptrace_sethbpregs(child, addr,
+ (unsigned long __user *)data);
+ break;
+#endif
+
default:
ret = ptrace_request(child, request, addr, data);
break;
diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
index e0430d036cea..336f14e0e5c2 100644
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -36,6 +36,7 @@
#include <asm/procinfo.h>
#include <asm/sections.h>
#include <asm/setup.h>
+#include <asm/smp_plat.h>
#include <asm/mach-types.h>
#include <asm/cacheflush.h>
#include <asm/cachetype.h>
@@ -524,7 +525,7 @@ request_standard_resources(struct meminfo *mi, struct machine_desc *mdesc)
kernel_code.start = virt_to_phys(_text);
kernel_code.end = virt_to_phys(_etext - 1);
- kernel_data.start = virt_to_phys(_data);
+ kernel_data.start = virt_to_phys(_sdata);
kernel_data.end = virt_to_phys(_end - 1);
for (i = 0; i < mi->nr_banks; i++) {
@@ -859,7 +860,8 @@ void __init setup_arch(char **cmdline_p)
request_standard_resources(&meminfo, mdesc);
#ifdef CONFIG_SMP
- smp_init_cpus();
+ if (is_smp())
+ smp_init_cpus();
#endif
reserve_crashkernel();
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 40dc74f2b27f..32e16da5cbce 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -567,7 +567,8 @@ void smp_send_stop(void)
{
cpumask_t mask = cpu_online_map;
cpu_clear(smp_processor_id(), mask);
- send_ipi_message(&mask, IPI_CPU_STOP);
+ if (!cpus_empty(mask))
+ send_ipi_message(&mask, IPI_CPU_STOP);
}
/*
diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c
index dd81a918c106..2a161765f6d5 100644
--- a/arch/arm/kernel/unwind.c
+++ b/arch/arm/kernel/unwind.c
@@ -146,6 +146,8 @@ static struct unwind_idx *unwind_find_idx(unsigned long addr)
addr < table->end_addr) {
idx = search_index(addr, table->start,
table->stop - 1);
+ /* Move-to-front to exploit common traces */
+ list_move(&table->list, &unwind_tables);
break;
}
}
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index b16c07914b55..065d35de0e01 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -40,6 +40,11 @@ SECTIONS
__tagtable_begin = .;
*(.taglist.init)
__tagtable_end = .;
+#ifdef CONFIG_SMP_ON_UP
+ __smpalt_begin = .;
+ *(.alt.smp.init)
+ __smpalt_end = .;
+#endif
INIT_SETUP(16)
@@ -104,8 +109,6 @@ SECTIONS
RO_DATA(PAGE_SIZE)
- _etext = .; /* End of text and rodata section */
-
#ifdef CONFIG_ARM_UNWIND
/*
* Stack unwinding tables
@@ -123,6 +126,8 @@ SECTIONS
}
#endif
+ _etext = .; /* End of text and rodata section */
+
#ifdef CONFIG_XIP_KERNEL
__data_loc = ALIGN(4); /* location in binary */
. = PAGE_OFFSET + TEXT_OFFSET;
@@ -237,6 +242,12 @@ SECTIONS
/* Default discards */
DISCARDS
+
+#ifndef CONFIG_SMP_ON_UP
+ /DISCARD/ : {
+ *(.alt.smp.init)
+ }
+#endif
}
/*
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-17 11:57:51 +0000