Make kprobes to accept 1-level nesting instead of missing it
on arm64.
Any kprobes hits in kprobes pre/post handler context can be
nested at once. If the other kprobes hits in the nested pre/post
handler context, that will be missed.
We can test this feature on the kernel with
CONFIG_KPROBE_EVENTS_ON_NOTRACE=y as below.
# cd /sys/kernel/debug/tracing
# echo p ring_buffer_lock_reserve > kprobe_events
# echo p vfs_read >> kprobe_events
# echo stacktrace > events/kprobes/p_ring_buffer_lock_reserve_0/trigger
# echo 1 > events/kprobes/enable
# cat trace
...
sh-211 [005] d..2 71.708242: p_ring_buffer_lock_reserve_0: (ring_buffer_lock_reserve+0x0/0x4c8)
sh-211 [005] d..2 71.709982: <stack trace>
=> kprobe_dispatcher
=> kprobe_breakpoint_handler
=> call_break_hook
=> brk_handler
=> do_debug_exception
=> el1_sync_handler
=> el1_sync
=> ring_buffer_lock_reserve
=> kprobe_trace_func
=> kprobe_dispatcher
=> kprobe_breakpoint_handler
=> call_break_hook
=> brk_handler
=> do_debug_exception
=> el1_sync_handler
=> el1_sync
=> vfs_read
=> __arm64_sys_read
=> el0_svc_common.constprop.3
=> do_el0_svc
=> el0_sync_handler
=> el0_sync
This shows brk_handler is nested.
Signed-off-by: Masami Hiramatsu <mhiramat@xxxxxxxxxx>
---
arch/arm64/include/asm/kprobes.h | 5 ++
arch/arm64/kernel/probes/kprobes.c | 75 ++++++++++++++++++++----------------
2 files changed, 46 insertions(+), 34 deletions(-)
diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h
index 97e511d645a2..b2ebd3bad794 100644
--- a/arch/arm64/include/asm/kprobes.h
+++ b/arch/arm64/include/asm/kprobes.h
@@ -34,12 +34,15 @@ struct kprobe_step_ctx {
unsigned long match_addr;
};
+#define KPROBE_NEST_MAX 2
+
/* per-cpu kprobe control block */
struct kprobe_ctlblk {
unsigned int kprobe_status;
unsigned long saved_irqflag;
- struct prev_kprobe prev_kprobe;
+ struct prev_kprobe prev[KPROBE_NEST_MAX];
struct kprobe_step_ctx ss_ctx;
+ int nested;
};
void arch_remove_kprobe(struct kprobe *);
diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index d1c95dcf1d78..27d52726277b 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -153,14 +153,18 @@ void __kprobes arch_remove_kprobe(struct kprobe *p)
static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
{
- kcb->prev_kprobe.kp = kprobe_running();
- kcb->prev_kprobe.status = kcb->kprobe_status;
+ int i = kcb->nested++;
+
+ kcb->prev[i].kp = kprobe_running();
+ kcb->prev[i].status = kcb->kprobe_status;
}
static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
{
- __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
- kcb->kprobe_status = kcb->prev_kprobe.status;
+ int i = --kcb->nested;
+
+ __this_cpu_write(current_kprobe, kcb->prev[i].kp);
+ kcb->kprobe_status = kcb->prev[i].status;
}
static void __kprobes set_current_kprobe(struct kprobe *p)
@@ -168,6 +172,14 @@ static void __kprobes set_current_kprobe(struct kprobe *p)
__this_cpu_write(current_kprobe, p);
}
+static nokprobe_inline void pop_current_kprobe(struct kprobe_ctlblk *kcb)
+{
+ if (kcb->nested)
+ restore_previous_kprobe(kcb);
+ else
+ reset_current_kprobe();
+}
+
/*
* Interrupts need to be disabled before single-step mode is set, and not
* reenabled until after single-step mode ends.
@@ -243,6 +255,10 @@ static int __kprobes reenter_kprobe(struct kprobe *p,
switch (kcb->kprobe_status) {
case KPROBE_HIT_SSDONE:
case KPROBE_HIT_ACTIVE:
+ if (kcb->nested < KPROBE_NEST_MAX - 1) {
+ save_previous_kprobe(kcb);
+ return 0;
+ }
kprobes_inc_nmissed_count(p);
setup_singlestep(p, regs, kcb, 1);
break;
@@ -286,7 +302,7 @@ post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
cur->post_handler(cur, regs, 0);
}
- reset_current_kprobe();
+ pop_current_kprobe(kcb);
}
int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
@@ -309,11 +325,7 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
BUG();
kernel_disable_single_step();
-
- if (kcb->kprobe_status == KPROBE_REENTER)
- restore_previous_kprobe(kcb);
- else
- reset_current_kprobe();
+ pop_current_kprobe(kcb);
break;
case KPROBE_HIT_ACTIVE:
@@ -357,30 +369,27 @@ static void __kprobes kprobe_handler(struct pt_regs *regs)
p = get_kprobe((kprobe_opcode_t *) addr);
if (p) {
- if (cur_kprobe) {
- if (reenter_kprobe(p, regs, kcb))
- return;
- } else {
- /* Probe hit */
- set_current_kprobe(p);
- kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+ if (cur_kprobe && reenter_kprobe(p, regs, kcb))
+ return;
+ /* Probe hit */
+ set_current_kprobe(p);
+ kcb->kprobe_status = KPROBE_HIT_ACTIVE;
- /*
- * If we have no pre-handler or it returned 0, we
- * continue with normal processing. If we have a
- * pre-handler and it returned non-zero, it will
- * modify the execution path and no need to single
- * stepping. Let's just reset current kprobe and exit.
- *
- * pre_handler can hit a breakpoint and can step thru
- * before return, keep PSTATE D-flag enabled until
- * pre_handler return back.
- */
- if (!p->pre_handler || !p->pre_handler(p, regs)) {
- setup_singlestep(p, regs, kcb, 0);
- } else
- reset_current_kprobe();
- }
+ /*
+ * If we have no pre-handler or it returned 0, we
+ * continue with normal processing. If we have a
+ * pre-handler and it returned non-zero, it will
+ * modify the execution path and no need to single
+ * stepping. Let's just reset current kprobe and exit.
+ *
+ * pre_handler can hit a breakpoint and can step thru
+ * before return, keep PSTATE D-flag enabled until
+ * pre_handler return back.
+ */
+ if (!p->pre_handler || !p->pre_handler(p, regs)) {
+ setup_singlestep(p, regs, kcb, 0);
+ } else
+ pop_current_kprobe(kcb);
}
/*
* The breakpoint instruction was removed right
