@@ -722,6 +723,91 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
*pprog = prog;
}
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() for the
+ * trampoline, see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the equivalent
+ * macros in arch/x86/kernel/ftrace_64.S. The image is not relocated, so
+ * current_task and rcu_tasks_trace_srcu_struct are referenced by absolute
+ * (sign-extended 32-bit) address, the form the JIT already relies on for
+ * this_cpu_off. Uses r10 and r11, which are scratch at every emission
+ * point, and clobbers flags.
+ *
+ * lock: unlock:
+ * mov r11, gs:[current_task] mov r11, gs:[current_task]
+ * mov r10d, [r11+nesting_off] mov r10d, [r11+nesting_off]
+ * inc dword ptr [r11+nesting_off] sub r10d, 1
+ * test r10d, r10d jnz 2f
+ * jnz 1f mov r10, [r11+scp_off]
+ * mov r10, [&srcu.srcu_ctrp] mov dword ptr [r11+nesting_off], 0
+ * inc qword ptr gs:[r10+locks_off] (smp_mb)
+ * mov [r11+scp_off], r10 inc qword ptr gs:[r10+unlocks_off]
+ * (smp_mb) jmp 3f
+ * 1: 2: mov [r11+nesting_off], r10d
+ * 3:
+ */
+static void emit_trace_rcu_reader(u8 **pprog, bool lock)
+{
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ const u32 nesting_off = offsetof(struct task_struct, trc_reader_nesting);
+ const u32 scp_off = offsetof(struct task_struct, trc_reader_scp);
+ const u32 locks_off = offsetof(struct srcu_ctr, srcu_locks);
+ const u32 unlocks_off = offsetof(struct srcu_ctr, srcu_unlocks);
+ const bool mb = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB);
+ u8 *prog = *pprog;
+
+ /* The plain this_cpu_inc() form of __srcu_read_lock_fast(). */
+ BUILD_BUG_ON(IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+
+ /* mov r11, gs:[abs32 current_task] */
+ EMIT2(0x65, 0x4C);
+ EMIT3(0x8B, 0x1C, 0x25);
+ EMIT((u32)(unsigned long)¤t_task, 4);
+ /* mov r10d, dword ptr [r11 + nesting_off] */
+ EMIT3_off32(0x45, 0x8B, 0x93, nesting_off);
+
+ if (lock) {
+ /* inc dword ptr [r11 + nesting_off] */
+ EMIT3_off32(0x41, 0xFF, 0x83, nesting_off);
+ /* test r10d, r10d */
+ EMIT3(0x45, 0x85, 0xD2);
+ /* jnz 1f */
+ EMIT2(X86_JNE, 8 + 8 + 7 + (mb ? 6 : 0));
+ /* mov r10, qword ptr [abs32 &rcu_tasks_trace_srcu_struct.srcu_ctrp] */
+ EMIT4(0x4C, 0x8B, 0x14, 0x25);
+ EMIT((u32)(unsigned long)&rcu_tasks_trace_srcu_struct.srcu_ctrp, 4);
+ /* inc qword ptr gs:[r10 + locks_off] */
+ EMIT4_off32(0x65, 0x49, 0xFF, 0x82, locks_off);
+ /* mov qword ptr [r11 + scp_off], r10 */
+ EMIT3_off32(0x4D, 0x89, 0x93, scp_off);
+ /* smp_mb(): lock add dword ptr [rsp - 4], 0 */
+ if (mb)
+ EMIT2_off32(0xF0, 0x83, 0x00FC2444);
+ /* 1: */
+ } else {
+ /* sub r10d, 1 */
+ EMIT4(0x41, 0x83, 0xEA, 0x01);
+ /* jnz 2f */
+ EMIT2(X86_JNE, 7 + 11 + (mb ? 6 : 0) + 8 + 2);
+ /* mov r10, qword ptr [r11 + scp_off] */
+ EMIT3_off32(0x4D, 0x8B, 0x93, scp_off);
+ /* mov dword ptr [r11 + nesting_off], 0 */
+ EMIT3_off32(0x41, 0xC7, 0x83, nesting_off);
+ EMIT(0, 4);
+ if (mb)
+ EMIT2_off32(0xF0, 0x83, 0x00FC2444);
+ /* inc qword ptr gs:[r10 + unlocks_off] */
+ EMIT4_off32(0x65, 0x49, 0xFF, 0x82, unlocks_off);
+ /* jmp 3f */
+ EMIT2(0xEB, 7);
+ /* 2: mov dword ptr [r11 + nesting_off], r10d */
+ EMIT3_off32(0x45, 0x89, 0x93, nesting_off);
+ /* 3: */
+ }
+
+ *pprog = prog;
+#endif
+}
+
static void emit_return(u8 **pprog, u8 *ip)
{
u8 *prog = *pprog;@@ -3610,6 +3696,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
/* mov QWORD PTR [rbp - rbx_off], rbx */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off);
+ /*
+ * Tasks RCU keeps this image alive only while we are a Tasks Trace
+ * reader; the instructions before this point (and after the final
+ * unlock) are covered by the irq-exit IP check. One reader spans
+ * __bpf_tramp_enter() and the fentry/fmod_ret progs, a second one
+ * the fexit progs and __bpf_tramp_exit(); the original function runs
+ * outside both, with the image pinned by im->pcref instead.
+ */
+ emit_trace_rcu_reader(&prog, true);
+
func_meta = nr_regs;
/* Store number of argument registers of the traced function */
emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta);
@@ -3660,6 +3756,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
+ emit_trace_rcu_reader(&prog, false);
restore_regs(m, &prog, regs_off);
save_args(m, &prog, arg_stack_off, true, flags, 0);
@@ -3682,6 +3779,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
}
/* remember return value in a stack for bpf prog to access */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
+ /*
+ * Second reader. Taken before ip_after_call so that the
+ * ip_after_call -> ip_epilogue jump patched in at teardown is
+ * inside it too; the fmod_ret early exit jumps past this still
+ * holding the first reader, so either way exactly one is held.
+ */
+ emit_trace_rcu_reader(&prog, true);
im->ip_after_call = image + (prog - (u8 *)rw_image);
emit_nops(&prog, X86_PATCH_SIZE);
}