1 // SPDX-License-Identifier: GPL-2.0-or-later
17 #include <asm/text-patching.h>
34 			".global optprobe_template_entry\n"
36 			".global optprobe_template_sub_sp\n"
39 			"	stmia	sp, {r0 - r14} \n"
40 			".global optprobe_template_add_sp\n"
50 			 * AEABI requires an 8-bytes alignment stack. If
69 			".global optprobe_template_restore_begin\n"
71 			"	ldmia	sp, {r0 - r15}\n"
72 			".global optprobe_template_restore_orig_insn\n"
75 			".global optprobe_template_restore_end\n"
78 			".global optprobe_template_val\n"
81 			".global optprobe_template_call\n"
84 			".global optprobe_template_end\n"
88 	((unsigned long *)optprobe_template_val - (unsigned long *)optprobe_template_entry)
90 	((unsigned long *)optprobe_template_call - (unsigned long *)optprobe_template_entry)
92 	((unsigned long *)optprobe_template_end - (unsigned long *)optprobe_template_entry)
94 	((unsigned long *)optprobe_template_add_sp - (unsigned long *)optprobe_template_entry)
96 	((unsigned long *)optprobe_template_sub_sp - (unsigned long *)optprobe_template_entry)
98 	((unsigned long *)optprobe_template_restore_begin - (unsigned long *)optprobe_template_entry)
100 	((unsigned long *)optprobe_template_restore_orig_insn - (unsigned long *)optprobe_template_entry)
102 	((unsigned long *)optprobe_template_restore_end - (unsigned long *)optprobe_template_entry)
111 	return optinsn->insn != NULL;  in arch_prepared_optinsn()
127 	if (kp->ainsn.stack_space < 0)  in can_optimize()
133 	if (kp->ainsn.stack_space > 255 - sizeof(struct pt_regs))  in can_optimize()
142 	if (op->optinsn.insn) {  in __arch_remove_optimized_kprobe()
143 		free_optinsn_slot(op->optinsn.insn, dirty);  in __arch_remove_optimized_kprobe()
144 		op->optinsn.insn = NULL;  in __arch_remove_optimized_kprobe()
149 optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)  in optimized_callback()  argument
152 	struct kprobe *p = &op->kp;  in optimized_callback()
156 	regs->ARM_pc = (unsigned long)op->kp.addr;  in optimized_callback()
157 	regs->ARM_ORIG_r0 = ~0UL;  in optimized_callback()
163 		kprobes_inc_nmissed_count(&op->kp);  in optimized_callback()
165 		__this_cpu_write(current_kprobe, &op->kp);  in optimized_callback()
166 		kcb->kprobe_status = KPROBE_HIT_ACTIVE;  in optimized_callback()
167 		opt_pre_handler(&op->kp, regs);  in optimized_callback()
175 	if (!p->ainsn.kprobe_direct_exec)  in optimized_callback()
176 		op->kp.ainsn.insn_singlestep(p->opcode, &p->ainsn, regs);  in optimized_callback()
190 		return -EILSEQ;  in NOKPROBE_SYMBOL()
194 		return -ENOMEM;  in NOKPROBE_SYMBOL()
204 	 *  +------+---+---+---+---+----------------+  in NOKPROBE_SYMBOL()
206 	 *  +------+---+---+---+---+----------------+  in NOKPROBE_SYMBOL()
214 	 *   (0xff800000 << 2) = 0xfe000000 = -0x2000000  in NOKPROBE_SYMBOL()
221 	rel_chk = (unsigned long)((long)code -  in NOKPROBE_SYMBOL()
222 			(long)orig->addr + 8) & 0xfe000003;  in NOKPROBE_SYMBOL()
231 		return -ERANGE;  in NOKPROBE_SYMBOL()
234 	/* Copy arch-dep-instance from template. */  in NOKPROBE_SYMBOL()
239 	BUG_ON(orig->ainsn.stack_space < 0);  in NOKPROBE_SYMBOL()
241 	stack_protect += orig->ainsn.stack_space;  in NOKPROBE_SYMBOL()
260 	orig->ainsn.kprobe_direct_exec = false;  in NOKPROBE_SYMBOL()
261 	if (can_kprobe_direct_exec(orig->ainsn.register_usage_flags)) {  in NOKPROBE_SYMBOL()
264 				(unsigned long)(op->kp.addr) + 4);  in NOKPROBE_SYMBOL()
267 			 * Replace original 'ldmia sp, {r0 - r15}' with  in NOKPROBE_SYMBOL()
268 			 * 'ldmia {r0 - r14}', restore all registers except pc.  in NOKPROBE_SYMBOL()
273 			code[TMPL_RESTORE_ORIGN_INSN] = __opcode_to_mem_arm(orig->opcode);  in NOKPROBE_SYMBOL()
277 			orig->ainsn.kprobe_direct_exec = true;  in NOKPROBE_SYMBOL()
284 	/* Set op->optinsn.insn means prepared. */  in NOKPROBE_SYMBOL()
285 	op->optinsn.insn = code;  in NOKPROBE_SYMBOL()
295 		WARN_ON(kprobe_disabled(&op->kp));  in arch_optimize_kprobes()
301 		memcpy(op->optinsn.copied_insn, op->kp.addr,  in arch_optimize_kprobes()
304 		insn = arm_gen_branch((unsigned long)op->kp.addr,  in arch_optimize_kprobes()
305 				(unsigned long)op->optinsn.insn);  in arch_optimize_kprobes()
313 			  op->optinsn.copied_insn[0]) & 0xf0000000) |  in arch_optimize_kprobes()
321 		kprobes_remove_breakpoint(op->kp.addr, insn);  in arch_optimize_kprobes()
323 		list_del_init(&op->list);  in arch_optimize_kprobes()
329 	arch_arm_kprobe(&op->kp);  in arch_unoptimize_kprobe()
343 		list_move(&op->list, done_list);  in arch_unoptimize_kprobes()
350 	return (op->kp.addr <= addr &&  in arch_within_optimized_kprobe()
351 		op->kp.addr + (RELATIVEJUMP_SIZE / sizeof(kprobe_opcode_t)) > addr);  in arch_within_optimized_kprobe()