[PATCH RFC v3 02/13] rcu-tasks-trace: Inline rcu_read_lock_trace() and annotate inside the reader
flat view
COLD16d
From: Josef Bacik <josef@toxicpanda.com>
Date: 2026-09-15 13:17:51
Also in:
bpf, lkml, rcu, xen-devel
Subsystem:
read-copy update (rcu), sleepable read-copy update (srcu), the rest · Maintainers:
"Paul E. McKenney", Frederic Weisbecker, Neeraj Upadhyay, Joel Fernandes, Josh Triplett, Boqun Feng, Uladzislau Rezki, Lai Jiangshan, Onur Özkan, Linus Torvalds
rcu_read_lock_trace() calls rcu_try_lock_acquire() before it has entered the SRCU-fast reader, and rcu_read_unlock_trace() calls srcu_lock_release() after it has left it. rcu_read_lock() and rcu_read_unlock() do it the other way around, annotating strictly inside the critical section, and rcu_read_lock_tasks_trace() already follows that order on the lock side. Make the trace variants match. Also make them, and the __srcu_read_lock_fast() and __srcu_read_unlock_fast() they are built on, __always_inline like rcu_read_lock() rather than leaving it to the compiler, which does outline all four in KASAN/KCOV builds. Besides consistency, this means the first thing a caller of rcu_read_lock_trace() does is enter the reader and the last thing rcu_read_unlock_trace() does is leave it, with no out-of-line call on the outside. A later patch relies on that for callers whose own text is protected by the reader they are about to take. Assisted-by: LLM Signed-off-by: Josef Bacik <josef@toxicpanda.com> --- include/linux/rcupdate_trace.h | 22 ++++++++++------------ include/linux/srcutiny.h | 4 ++-- include/linux/srcutree.h | 5 +++-- 3 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h
index 273c59a03251..4035054309d7 100644
--- a/include/linux/rcupdate_trace.h
+++ b/include/linux/rcupdate_trace.h@@ -93,22 +93,20 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) * * For more details, please see the documentation for rcu_read_lock(). */ -static inline void rcu_read_lock_trace(void) +static __always_inline void rcu_read_lock_trace(void) { int n; struct task_struct *t = current; - rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); n = READ_ONCE(t->trc_reader_nesting); WRITE_ONCE(t->trc_reader_nesting, n + 1); - if (n) { - // In case we interrupted a Tasks Trace RCU reader. - return; - } - barrier(); // nesting before scp to protect against interrupt handler. - t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); - if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) - smp_mb(); // Placeholder for more selective ordering + if (!n) { + barrier(); // nesting before scp to protect against interrupt handler. + t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); + if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) + smp_mb(); // Placeholder for more selective ordering + } // Else we interrupted a Tasks Trace RCU reader. + rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); } /**
@@ -120,12 +118,13 @@ static inline void rcu_read_lock_trace(void) * * For more details, please see the documentation for rcu_read_unlock(). */ -static inline void rcu_read_unlock_trace(void) +static __always_inline void rcu_read_unlock_trace(void) { int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; + srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map); n = READ_ONCE(t->trc_reader_nesting) - 1; if (n) { WRITE_ONCE(t->trc_reader_nesting, n);
@@ -137,7 +136,6 @@ static inline void rcu_read_unlock_trace(void) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); } - srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map); } /**
diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index fbcf13bc12d1..a43bae11c81c 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h@@ -101,13 +101,13 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss return (struct srcu_ctr __percpu *)(intptr_t)idx; } -static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp) +static __always_inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp) { return __srcu_ctr_to_ptr(ssp, __srcu_read_lock(ssp)); } -static inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) +static __always_inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp) { __srcu_read_unlock(ssp, __srcu_ptr_to_ctr(ssp, scp));
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 75e54e4f963f..fdb42ab50301 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h@@ -286,7 +286,8 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss * on architectures that support NMIs but do not supply NMI-safe * implementations of this_cpu_inc(). */ -static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct srcu_struct *ssp) +static __always_inline struct srcu_ctr __percpu notrace * +__srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp) { struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);
@@ -309,7 +310,7 @@ static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct src * Please see the __srcu_read_lock_fast() function's header comment for * information on implicit RCU readers and NMI safety. */ -static inline void notrace +static __always_inline void notrace __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp) {
--
2.55.0