[RFC PATCH 4/9] arm64/kprobes: Make the single-step machinery noinstr
From: Hongyan Xia <hidden>
Date: 2026-07-27 12:26:27
Also in:
lkml
Subsystem:
arm64 port (aarch64 architecture), the rest · Maintainers:
Catalin Marinas, Will Deacon, Linus Torvalds
From: Hongyan Xia <redacted> Convert the core of the arm64 kprobe single-step flow to noinstr: save_previous_kprobe(), restore_previous_kprobe(), set_current_kprobe(), kprobes_save_local_irqflag(), kprobes_restore_local_irqflag(), setup_singlestep(), reenter_kprobe() and post_kprobe_handler(). These functions only touch per-cpu state, pt_regs and DAIF, except for: - kprobes_inc_nmissed_count(), which is generic and instrumentable; add an arm64-local wrapper that calls it bounded by instrumentation_begin()/end(); - the instruction simulation path in setup_singlestep(), whose decode handlers are instrumentable; bound arch_simulate_insn() with an instrumentation window; - the unrecoverable-reentry pr_warn()/dump_kprobe()/BUG() path and the default WARN_ON(), both bounded with instrumentation windows. The __kprobes attribute (notrace + .kprobes.text) is replaced by noinstr, which is a strict superset for these functions. Signed-off-by: Hongyan Xia <redacted> --- arch/arm64/kernel/probes/kprobes.c | 39 ++++++++++++++++++------------ 1 file changed, 24 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index 4e0efad5caf2..1b12341b2af3 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c@@ -14,6 +14,7 @@ #include <linux/extable.h> #include <linux/kasan.h> #include <linux/kernel.h> +#include <linux/instrumentation.h> #include <linux/kprobes.h> #include <linux/sched/debug.h> #include <linux/set_memory.h>
@@ -39,7 +40,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -static void __kprobes +static void noinstr post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *); void *alloc_insn_page(void)
@@ -170,7 +171,7 @@ void __kprobes arch_remove_kprobe(struct kprobe *p) } } -static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +static void noinstr save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status;
@@ -184,7 +185,7 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } -static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +static void noinstr restore_previous_kprobe(struct kprobe_ctlblk *kcb) { __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status;
@@ -197,7 +198,7 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } -static void __kprobes set_current_kprobe(struct kprobe *p) +static void noinstr set_current_kprobe(struct kprobe *p) { __this_cpu_write(current_kprobe, p); }
@@ -207,23 +208,23 @@ static void __kprobes set_current_kprobe(struct kprobe *p) * simple and avoid nesting exceptions. Interrupts do have to be disabled since * the kprobe state is per-CPU and doesn't get migrated. */ -static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, - struct pt_regs *regs) +static void noinstr kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) { kcb->saved_irqflag = regs->pstate & DAIF_MASK; regs->pstate |= DAIF_MASK; } -static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, - struct pt_regs *regs) +static void noinstr kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) { regs->pstate &= ~DAIF_MASK; regs->pstate |= kcb->saved_irqflag; } -static void __kprobes setup_singlestep(struct kprobe *p, - struct pt_regs *regs, - struct kprobe_ctlblk *kcb, int reenter) +static void noinstr setup_singlestep(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) { unsigned long slot;
@@ -244,13 +245,15 @@ static void __kprobes setup_singlestep(struct kprobe *p, instruction_pointer_set(regs, slot); } else { /* insn simulation */ + instrumentation_begin(); arch_simulate_insn(p, regs); + instrumentation_end(); } } -static int __kprobes reenter_kprobe(struct kprobe *p, - struct pt_regs *regs, - struct kprobe_ctlblk *kcb) +static int noinstr reenter_kprobe(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) { switch (kcb->kprobe_status) { case KPROBE_HIT_SSDONE:
@@ -262,23 +265,29 @@ static int __kprobes reenter_kprobe(struct kprobe *p, * recoverable one-level reentry, so handle it in the same way as * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. */ + instrumentation_begin(); kprobes_inc_nmissed_count(p); + instrumentation_end(); setup_singlestep(p, regs, kcb, 1); break; case KPROBE_REENTER: + instrumentation_begin(); pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); BUG(); + instrumentation_end(); break; default: + instrumentation_begin(); WARN_ON(1); + instrumentation_end(); return 0; } return 1; } -static void __kprobes +static void noinstr post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, struct pt_regs *regs) { /* return addr restore if non-branching insn */
--
2.47.3