[PATCH RFC 11/13] samples: ftrace: Maintain Tasks RCU trampoline nesting in direct-call trampolines
From: Josef Bacik <josef@toxicpanda.com>
Date: 2026-09-10 18:51:17
Also in:
bpf, linux-arm-kernel, lkml, rcu, xen-devel
Subsystem:
the rest · Maintainer:
Linus Torvalds
Follow the register_ftrace_direct() contract in the sample modules: on x86-64 and arm64, have each hand-written trampoline increment current->rcu_tramp_nesting before calling its C handler and decrement it before returning, via a small shared samples/ftrace/ftrace-direct.h. %r11 and x12/w13 are used as scratch; both are caller-saved, non-argument registers and therefore dead on entry to and exit from an fentry trampoline. The header pulls in the generated asm-offsets.h only on those two architectures, since it is not generally safe to include from C (PPC32's TASK_SIZE and arm64's TRAMP_VALIAS clash with the C definitions; the latter is worked around locally with push_macro/pop_macro). Other architectures get empty macros and are unchanged. Assisted-by: LLM Signed-off-by: Josef Bacik <josef@toxicpanda.com> --- samples/ftrace/ftrace-direct-modify.c | 9 ++++ samples/ftrace/ftrace-direct-multi-modify.c | 9 ++++ samples/ftrace/ftrace-direct-multi.c | 5 +++ samples/ftrace/ftrace-direct-too.c | 5 +++ samples/ftrace/ftrace-direct.c | 5 +++ samples/ftrace/ftrace-direct.h | 64 +++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+)
diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c
index 164d9dd6fd92..eb8230fa4242 100644
--- a/samples/ftrace/ftrace-direct-modify.c
+++ b/samples/ftrace/ftrace-direct-modify.c@@ -2,6 +2,7 @@ #include <linux/module.h> #include <linux/kthread.h> #include <linux/ftrace.h> +#include "ftrace-direct.h" #if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32) #include <asm/asm-offsets.h> #endif
@@ -73,7 +74,9 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " call my_direct_func1\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp1, .-my_tramp1\n"
@@ -85,7 +88,9 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " call my_direct_func2\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp2, .-my_tramp2\n"
@@ -141,11 +146,13 @@ asm ( " .globl my_tramp1\n" " my_tramp1:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #16\n" " stp x9, x30, [sp]\n" " bl my_direct_func1\n" " ldp x30, x9, [sp]\n" " add sp, sp, #16\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp1, .-my_tramp1\n"
@@ -153,11 +160,13 @@ asm ( " .globl my_tramp2\n" " my_tramp2:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #16\n" " stp x9, x30, [sp]\n" " bl my_direct_func2\n" " ldp x30, x9, [sp]\n" " add sp, sp, #16\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp2, .-my_tramp2\n" " .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c
index b03766c6217b..c8f1062e5d1a 100644
--- a/samples/ftrace/ftrace-direct-multi-modify.c
+++ b/samples/ftrace/ftrace-direct-multi-modify.c@@ -2,6 +2,7 @@ #include <linux/module.h> #include <linux/kthread.h> #include <linux/ftrace.h> +#include "ftrace-direct.h" #if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32) #include <asm/asm-offsets.h> #endif
@@ -77,10 +78,12 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " pushq %rdi\n" " movq 8(%rbp), %rdi\n" " call my_direct_func1\n" " popq %rdi\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp1, .-my_tramp1\n"
@@ -92,10 +95,12 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " pushq %rdi\n" " movq 8(%rbp), %rdi\n" " call my_direct_func2\n" " popq %rdi\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp2, .-my_tramp2\n"
@@ -154,6 +159,7 @@ asm ( " .globl my_tramp1\n" " my_tramp1:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #32\n" " stp x9, x30, [sp]\n" " str x0, [sp, #16]\n"
@@ -162,6 +168,7 @@ asm ( " ldp x30, x9, [sp]\n" " ldr x0, [sp, #16]\n" " add sp, sp, #32\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp1, .-my_tramp1\n"
@@ -169,6 +176,7 @@ asm ( " .globl my_tramp2\n" " my_tramp2:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #32\n" " stp x9, x30, [sp]\n" " str x0, [sp, #16]\n"
@@ -177,6 +185,7 @@ asm ( " ldp x30, x9, [sp]\n" " ldr x0, [sp, #16]\n" " add sp, sp, #32\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp2, .-my_tramp2\n" " .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-multi.c b/samples/ftrace/ftrace-direct-multi.c
index 3fe6ddaf0b69..bc6a88dd4ffc 100644
--- a/samples/ftrace/ftrace-direct-multi.c
+++ b/samples/ftrace/ftrace-direct-multi.c@@ -3,6 +3,7 @@ #include <linux/mm.h> /* for handle_mm_fault() */ #include <linux/ftrace.h> +#include "ftrace-direct.h" #include <linux/sched/stat.h> #if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32) #include <asm/asm-offsets.h>
@@ -56,10 +57,12 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " pushq %rdi\n" " movq 8(%rbp), %rdi\n" " call my_direct_func\n" " popq %rdi\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp, .-my_tramp\n"
@@ -101,6 +104,7 @@ asm ( " .globl my_tramp\n" " my_tramp:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #32\n" " stp x9, x30, [sp]\n" " str x0, [sp, #16]\n"
@@ -109,6 +113,7 @@ asm ( " ldp x30, x9, [sp]\n" " ldr x0, [sp, #16]\n" " add sp, sp, #32\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp, .-my_tramp\n" " .popsection\n"
diff --git a/samples/ftrace/ftrace-direct-too.c b/samples/ftrace/ftrace-direct-too.c
index bf2411aa6fd7..247e418644a2 100644
--- a/samples/ftrace/ftrace-direct-too.c
+++ b/samples/ftrace/ftrace-direct-too.c@@ -3,6 +3,7 @@ #include <linux/mm.h> /* for handle_mm_fault() */ #include <linux/ftrace.h> +#include "ftrace-direct.h" #if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32) #include <asm/asm-offsets.h> #endif
@@ -61,6 +62,7 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " pushq %rdi\n" " pushq %rsi\n" " pushq %rdx\n"
@@ -70,6 +72,7 @@ asm ( " popq %rdx\n" " popq %rsi\n" " popq %rdi\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp, .-my_tramp\n"
@@ -110,6 +113,7 @@ asm ( " .globl my_tramp\n" " my_tramp:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #48\n" " stp x9, x30, [sp]\n" " stp x0, x1, [sp, #16]\n"
@@ -119,6 +123,7 @@ asm ( " ldp x0, x1, [sp, #16]\n" " ldp x2, x3, [sp, #32]\n" " add sp, sp, #48\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp, .-my_tramp\n" " .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.c b/samples/ftrace/ftrace-direct.c
index 5368c8c39cbb..9e1964baf28b 100644
--- a/samples/ftrace/ftrace-direct.c
+++ b/samples/ftrace/ftrace-direct.c@@ -3,6 +3,7 @@ #include <linux/sched.h> /* for wake_up_process() */ #include <linux/ftrace.h> +#include "ftrace-direct.h" #if !defined(CONFIG_ARM64) && !defined(CONFIG_PPC32) #include <asm/asm-offsets.h> #endif
@@ -54,9 +55,11 @@ asm ( " pushq %rbp\n" " movq %rsp, %rbp\n" CALL_DEPTH_ACCOUNT + RCU_TASKS_TRAMP_ENTER " pushq %rdi\n" " call my_direct_func\n" " popq %rdi\n" + RCU_TASKS_TRAMP_EXIT " leave\n" ASM_RET " .size my_tramp, .-my_tramp\n"
@@ -97,6 +100,7 @@ asm ( " .globl my_tramp\n" " my_tramp:" " hint 34\n" // bti c + RCU_TASKS_TRAMP_ENTER " sub sp, sp, #32\n" " stp x9, x30, [sp]\n" " str x0, [sp, #16]\n"
@@ -104,6 +108,7 @@ asm ( " ldp x30, x9, [sp]\n" " ldr x0, [sp, #16]\n" " add sp, sp, #32\n" + RCU_TASKS_TRAMP_EXIT " ret x9\n" " .size my_tramp, .-my_tramp\n" " .popsection\n"
diff --git a/samples/ftrace/ftrace-direct.h b/samples/ftrace/ftrace-direct.h
new file mode 100644
index 000000000000..d0313f33f47f
--- /dev/null
+++ b/samples/ftrace/ftrace-direct.h@@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _SAMPLES_FTRACE_DIRECT_H +#define _SAMPLES_FTRACE_DIRECT_H + +#include <linux/stringify.h> + +/* + * A direct-call trampoline is entered with no lock, refcount or RCU marker + * held; only Tasks RCU keeps it (and, for a module, its text) alive while a + * task is inside it or preempted in something it called. On architectures + * that select ARCH_HAS_RCU_TASKS_PREEMPT_QS a preemption is a Tasks RCU + * quiescent state unless current->rcu_tramp_nesting is non-zero, so the + * trampoline must raise it before calling out and drop it afterwards, exactly + * like the ftrace and BPF trampolines do. See rcu_tasks_trampoline_enter() + * and register_ftrace_direct(). The instructions before the increment and + * after the decrement are covered by ftrace_direct_mark_module(). + * + * These expand to instruction strings for use inside the samples' asm() + * trampolines. The scratch register is caller-saved and not an argument + * register, so it is dead on entry to and exit from an fentry trampoline. + * + * The generated asm-offsets.h is only pulled in on the architectures that need + * it here: it is not generally safe to include from C (e.g. PPC32's TASK_SIZE + * and arm64's TRAMP_VALIAS clash with the C definitions), which is why the + * samples themselves guard their own include of it. + */ +#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_X86_64) + +#include <asm/asm-offsets.h> + +#define RCU_TASKS_TRAMP_ENTER \ + " movq %gs:current_task(%rip), %r11\n" \ + " incl " __stringify(TASK_rcu_tramp_nesting) "(%r11)\n" +#define RCU_TASKS_TRAMP_EXIT \ + " movq %gs:current_task(%rip), %r11\n" \ + " decl " __stringify(TASK_rcu_tramp_nesting) "(%r11)\n" + +#elif defined(CONFIG_TASKS_RCU) && defined(CONFIG_ARM64) + +/* arm64's asm-offsets.h redefines TRAMP_VALIAS from <asm/fixmap.h>. */ +#pragma push_macro("TRAMP_VALIAS") +#undef TRAMP_VALIAS +#include <asm/asm-offsets.h> +#pragma pop_macro("TRAMP_VALIAS") + +#define RCU_TASKS_TRAMP_ENTER \ + " mrs x12, sp_el0\n" \ + " ldr w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n" \ + " add w13, w13, #1\n" \ + " str w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n" +#define RCU_TASKS_TRAMP_EXIT \ + " mrs x12, sp_el0\n" \ + " ldr w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n" \ + " sub w13, w13, #1\n" \ + " str w13, [x12, #" __stringify(TSK_RCU_TRAMP_NESTING) "]\n" + +#else + +#define RCU_TASKS_TRAMP_ENTER +#define RCU_TASKS_TRAMP_EXIT + +#endif + +#endif /* _SAMPLES_FTRACE_DIRECT_H */
--
2.55.0