summaryrefslogtreecommitdiffstats
path: root/arch/arm/probes/kprobes/opt-arm.c
blob: 7a449df0b359135e774badd06cdc53dc2aad3fb8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Kernel Probes Jump Optimization (Optprobes)
 *
 * Copyright (C) IBM Corporation, 2002, 2004
 * Copyright (C) Hitachi Ltd., 2012
 * Copyright (C) Huawei Inc., 2014
 */

#include <linux/kprobes.h>
#include <linux/jump_label.h>
#include <asm/kprobes.h>
#include <asm/cacheflush.h>
/* for arm_gen_branch */
#include <asm/insn.h>
/* for patch_text */
#include <asm/patch.h>

#include "core.h"

/*
 * See register_usage_flags. If the probed instruction doesn't use PC,
 * we can copy it into template and have it executed directly without
 * simulation or emulation.
 */
#define ARM_REG_PC	15
#define can_kprobe_direct_exec(m)	(!test_bit(ARM_REG_PC, &(m)))

/*
 * NOTE: the first sub and add instruction will be modified according
 * to the stack cost of the instruction.
 */
asm (
			".global optprobe_template_entry\n"
			"optprobe_template_entry:\n"
			".global optprobe_template_sub_sp\n"
			"optprobe_template_sub_sp:"
			"	sub	sp, sp, #0xff\n"
			"	stmia	sp, {r0 - r14} \n"
			".global optprobe_template_add_sp\n"
			"optprobe_template_add_sp:"
			"	add	r3, sp, #0xff\n"
			"	str	r3, [sp, #52]\n"
			"	mrs	r4, cpsr\n"
			"	str	r4, [sp, #64]\n"
			"	mov	r1, sp\n"
			"	ldr	r0, 1f\n"
			"	ldr	r2, 2f\n"
			/*
			 * AEABI requires an 8-bytes alignment stack. If
			 * SP % 8 != 0 (SP % 4 == 0 should be ensured),
			 * alloc more bytes here.
			 */
			"	and	r4, sp, #4\n"
			"	sub	sp, sp, r4\n"
#if __LINUX_ARM_ARCH__ >= 5
			"	blx	r2\n"
#else
			"	mov     lr, pc\n"
			"	mov	pc, r2\n"
#endif
			"	add	sp, sp, r4\n"
			"	ldr	r1, [sp, #64]\n"
			"	tst	r1, #"__stringify(PSR_T_BIT)"\n"
			"	ldrne	r2, [sp, #60]\n"
			"	orrne	r2, #1\n"
			"	strne	r2, [sp, #60] @ set bit0 of PC for thumb\n"
			"	msr	cpsr_cxsf, r1\n"
			".global optprobe_template_restore_begin\n"
			"optprobe_template_restore_begin:\n"
			"	ldmia	sp, {r0 - r15}\n"
			".global optprobe_template_restore_orig_insn\n"
			"optprobe_template_restore_orig_insn:\n"
			"	nop\n"
			".global optprobe_template_restore_end\n"
			"optprobe_template_restore_end:\n"
			"	nop\n"
			".global optprobe_template_val\n"
			"optprobe_template_val:\n"
			"1:	.long 0\n"
			".global optprobe_template_call\n"
			"optprobe_template_call:\n"
			"2:	.long 0\n"
			".global optprobe_template_end\n"
			"optprobe_template_end:\n");

#define TMPL_VAL_IDX \
	((unsigned long *)&optprobe_template_val - (unsigned long *)&optprobe_template_entry)
#define TMPL_CALL_IDX \
	((unsigned long *)&optprobe_template_call - (unsigned long *)&optprobe_template_entry)
#define TMPL_END_IDX \
	((unsigned long *)&optprobe_template_end - (unsigned long *)&optprobe_template_entry)
#define TMPL_ADD_SP \
	((unsigned long *)&optprobe_template_add_sp - (unsigned long *)&optprobe_template_entry)
#define TMPL_SUB_SP \
	((unsigned long *)&optprobe_template_sub_sp - (unsigned long *)&optprobe_template_entry)
#define TMPL_RESTORE_BEGIN \
	((unsigned long *)&optprobe_template_restore_begin - (unsigned long *)&optprobe_template_entry)
#define TMPL_RESTORE_ORIGN_INSN \
	((unsigned long *)&optprobe_template_restore_orig_insn - (unsigned long *)&optprobe_template_entry)
#define TMPL_RESTORE_END \
	((unsigned long *)&optprobe_template_restore_end - (unsigned long *)&optprobe_template_entry)

/*
 * ARM can always optimize an instruction when using ARM ISA, except
 * instructions like 'str r0, [sp, r1]' which store to stack and unable
 * to determine stack space consumption statically.
 */
int arch_prepared_optinsn(struct arch_optimized_insn *optinsn)
{
	return optinsn->insn != NULL;
}

/*
 * In ARM ISA, kprobe opt always replace one instruction (4 bytes
 * aligned and 4 bytes long). It is impossible to encounter another
 * kprobe in the address range. So always return 0.
 */
int arch_check_optimized_kprobe(struct optimized_kprobe *op)
{
	return 0;
}

/* Caller must ensure addr & 3 == 0 */
static int can_optimize(struct kprobe *kp)
{
	if (kp->ainsn.stack_space < 0)
		return 0;
	/*
	 * 255 is the biggest imm can be used in 'sub r0, r0, #<imm>'.
	 * Number larger than 255 needs special encoding.
	 */
	if (kp->ainsn.stack_space > 255 - sizeof(struct pt_regs))
		return 0;
	return 1;
}

/* Free optimized instruction slot */
static void
__arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
{
	if (op->optinsn.insn) {
		free_optinsn_slot(op->optinsn.insn, dirty);
		op->optinsn.insn = NULL;
	}
}

extern void kprobe_handler(struct pt_regs *regs);

static void
optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
{
	unsigned long flags;
	struct kprobe *p = &op->kp;
	struct kprobe_ctlblk *kcb;

	/* Save skipped registers */
	regs->ARM_pc = (unsigned long)op->kp.addr;
	regs->ARM_ORIG_r0 = ~0UL;

	local_irq_save(flags);
	kcb = get_kprobe_ctlblk();

	if (kprobe_running()) {
		kprobes_inc_nmissed_count(&op->kp);
	} else {
		__this_cpu_write(current_kprobe, &op->kp);
		kcb->kprobe_status = KPROBE_HIT_ACTIVE;
		opt_pre_handler(&op->kp, regs);
		__this_cpu_write(current_kprobe, NULL);
	}

	/*
	 * We singlestep the replaced instruction only when it can't be
	 * executed directly during restore.
	 */
	if (!p->ainsn.kprobe_direct_exec)
		op->kp.ainsn.insn_singlestep(p->opcode, &p->ainsn, regs);

	local_irq_restore(flags);
}
NOKPROBE_SYMBOL(optimized_callback)

int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *orig)
{
	kprobe_opcode_t *code;
	unsigned long rel_chk;
	unsigned long val;
	unsigned long stack_protect = sizeof(struct pt_regs);

	if (!can_optimize(orig))
		return -EILSEQ;

	code = get_optinsn_slot();
	if (!code)
		return -ENOMEM;

	/*
	 * Verify if the address gap is in 32MiB range, because this uses
	 * a relative jump.
	 *
	 * kprobe opt use a 'b' instruction to branch to optinsn.insn.
	 * According to ARM manual, branch instruction is:
	 *
	 *   31  28 27           24 23             0
	 *  +------+---+---+---+---+----------------+
	 *  | cond | 1 | 0 | 1 | 0 |      imm24     |
	 *  +------+---+---+---+---+----------------+
	 *
	 * imm24 is a signed 24 bits integer. The real branch offset is computed
	 * by: imm32 = SignExtend(imm24:'00', 32);
	 *
	 * So the maximum forward branch should be:
	 *   (0x007fffff << 2) = 0x01fffffc =  0x1fffffc
	 * The maximum backword branch should be:
	 *   (0xff800000 << 2) = 0xfe000000 = -0x2000000
	 *
	 * We can simply check (rel & 0xfe000003):
	 *  if rel is positive, (rel & 0xfe000000) shoule be 0
	 *  if rel is negitive, (rel & 0xfe000000) should be 0xfe000000
	 *  the last '3' is used for alignment checking.
	 */
	rel_chk = (unsigned long)((long)code -
			(long)orig->addr + 8) & 0xfe000003;

	if ((rel_chk != 0) && (rel_chk != 0xfe000000)) {
		/*
		 * Different from x86, we free code buf directly instead of
		 * calling __arch_remove_optimized_kprobe() because
		 * we have not fill any field in op.
		 */
		free_optinsn_slot(code, 0);
		return -ERANGE;
	}

	/* Copy arch-dep-instance from template. */
	memcpy(code, (unsigned long *)&optprobe_template_entry,
			TMPL_END_IDX * sizeof(kprobe_opcode_t));

	/* Adjust buffer according to instruction. */
	BUG_ON(orig->ainsn.stack_space < 0);

	stack_protect += orig->ainsn.stack_space;

	/* Should have been filtered by can_optimize(). */
	BUG_ON(stack_protect > 255);

	/* Create a 'sub sp, sp, #<stack_protect>' */
	code[TMPL_SUB_SP] = __opcode_to_mem_arm(0xe24dd000 | stack_protect);
	/* Create a 'add r3, sp, #<stack_protect>' */
	code[TMPL_ADD_SP] = __opcode_to_mem_arm(0xe28d3000 | stack_protect);

	/* Set probe information */
	val = (unsigned long)op;
	code[TMPL_VAL_IDX] = val;

	/* Set probe function call */
	val = (unsigned long)optimized_callback;
	code[TMPL_CALL_IDX] = val;

	/* If possible, copy insn and have it executed during restore */
	orig->ainsn.kprobe_direct_exec = false;
	if (can_kprobe_direct_exec(orig->ainsn.register_usage_flags)) {
		kprobe_opcode_t final_branch = arm_gen_branch(
				(unsigned long)(&code[TMPL_RESTORE_END]),
				(unsigned long)(op->kp.addr) + 4);
		if (final_branch != 0) {
			/*
			 * Replace original 'ldmia sp, {r0 - r15}' with
			 * 'ldmia {r0 - r14}', restore all registers except pc.
			 */
			code[TMPL_RESTORE_BEGIN] = __opcode_to_mem_arm(0xe89d7fff);

			/* The original probed instruction */
			code[TMPL_RESTORE_ORIGN_INSN] = __opcode_to_mem_arm(orig->opcode);

			/* Jump back to next instruction */
			code[TMPL_RESTORE_END] = __opcode_to_mem_arm(final_branch);
			orig->ainsn.kprobe_direct_exec = true;
		}
	}

	flush_icache_range((unsigned long)code,
			   (unsigned long)(&code[TMPL_END_IDX]));

	/* Set op->optinsn.insn means prepared. */
	op->optinsn.insn = code;
	return 0;
}

void __kprobes arch_optimize_kprobes(struct list_head *oplist)
{
	struct optimized_kprobe *op, *tmp;

	list_for_each_entry_safe(op, tmp, oplist, list) {
		unsigned long insn;
		WARN_ON(kprobe_disabled(&op->kp));

		/*
		 * Backup instructions which will be replaced
		 * by jump address
		 */
		memcpy(op->optinsn.copied_insn, op->kp.addr,
				RELATIVEJUMP_SIZE);

		insn = arm_gen_branch((unsigned long)op->kp.addr,
				(unsigned long)op->optinsn.insn);
		BUG_ON(insn == 0);

		/*
		 * Make it a conditional branch if replaced insn
		 * is consitional
		 */
		insn = (__mem_to_opcode_arm(
			  op->optinsn.copied_insn[0]) & 0xf0000000) |
			(insn & 0x0fffffff);

		/*
		 * Similar to __arch_disarm_kprobe, operations which
		 * removing breakpoints must be wrapped by stop_machine
		 * to avoid racing.
		 */
		kprobes_remove_breakpoint(op->kp.addr, insn);

		list_del_init(&op->list);
	}
}

void arch_unoptimize_kprobe(struct optimized_kprobe *op)
{
	arch_arm_kprobe(&op->kp);
}

/*
 * Recover original instructions and breakpoints from relative jumps.
 * Caller must call with locking kprobe_mutex.
 */
void arch_unoptimize_kprobes(struct list_head *oplist,
			    struct list_head *done_list)
{
	struct optimized_kprobe *op, *tmp;

	list_for_each_entry_safe(op, tmp, oplist, list) {
		arch_unoptimize_kprobe(op);
		list_move(&op->list, done_list);
	}
}

int arch_within_optimized_kprobe(struct optimized_kprobe *op,
				unsigned long addr)
{
	return ((unsigned long)op->kp.addr <= addr &&
		(unsigned long)op->kp.addr + RELATIVEJUMP_SIZE > addr);
}

void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
{
	__arch_remove_optimized_kprobe(op, 1);
}