[PATCH v2 1/3] rcu: introduce async mechanism for sync SRCU
From: Chao Leng <hidden>
Date: 2020-08-07 09:06:31
Also in:
linux-nvme
Subsystem:
read-copy update (rcu), sleepable read-copy update (srcu), the rest · Maintainers:
"Paul E. McKenney", Frederic Weisbecker, Neeraj Upadhyay, Joel Fernandes, Josh Triplett, Boqun Feng, Uladzislau Rezki, Lai Jiangshan, Linus Torvalds
In some scenarios we need sync lots of SRCUs together, if sync one by one, may need long time. Introduce async mechanism for sync SRCU: sync SRCUs together but no wait, and then wait until all SRCUs read-side critical-section complete. Thus reduce serial wait time. Signed-off-by: Chao Leng <redacted> --- include/linux/rcupdate_wait.h | 6 ++++++ include/linux/srcu.h | 1 + include/linux/srcutiny.h | 2 ++ include/linux/srcutree.h | 2 ++ kernel/rcu/srcutiny.c | 15 +++++++++++++++ kernel/rcu/srcutree.c | 26 ++++++++++++++++++++++++++ kernel/rcu/update.c | 11 +++++++++++ 7 files changed, 63 insertions(+)
diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h
index 699b938358bf..c8af1b8916f5 100644
--- a/include/linux/rcupdate_wait.h
+++ b/include/linux/rcupdate_wait.h@@ -16,7 +16,13 @@ struct rcu_synchronize { struct rcu_head head; struct completion completion; }; +struct rcu_synchronize_async { + struct rcu_head head; + atomic_t *count; +}; + void wakeme_after_rcu(struct rcu_head *head); +void wakeme_after_rcu_async(struct rcu_head *head); void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array, struct rcu_synchronize *rs_array);
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index e432cc92c73d..f657ce9f73b0 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h@@ -60,6 +60,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp); int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); void synchronize_srcu(struct srcu_struct *ssp); +void synchronize_srcu_async(struct srcu_struct *ssp, atomic_t *count); #ifdef CONFIG_DEBUG_LOCK_ALLOC
diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 5a5a1941ca15..54b257bc0ee5 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h@@ -11,6 +11,7 @@ #ifndef _LINUX_SRCU_TINY_H #define _LINUX_SRCU_TINY_H +#include <linux/rcupdate_wait.h> #include <linux/swait.h> struct srcu_struct {
@@ -23,6 +24,7 @@ struct srcu_struct { struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct work_struct srcu_work; /* For driving grace periods. */ + struct rcu_synchronize_async srcu_async; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index 9cfcc8a756ae..81ec38002e49 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h@@ -11,6 +11,7 @@ #ifndef _LINUX_SRCU_TREE_H #define _LINUX_SRCU_TREE_H +#include <linux/rcupdate_wait.h> #include <linux/rcu_node_tree.h> #include <linux/completion.h>
@@ -82,6 +83,7 @@ struct srcu_struct { /* callback for the barrier */ /* operation. */ struct delayed_work work; + struct rcu_synchronize_async srcu_async; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index 6208c1dae5c9..96bea76b78b8 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c@@ -190,6 +190,21 @@ void synchronize_srcu(struct srcu_struct *ssp) } EXPORT_SYMBOL_GPL(synchronize_srcu); +/* + * synchronize_srcu_async - do not wait until prior SRCU read-side + * critical-section completion, used for sync lots of SRCUs together + * to reduce wait time. + * Caution: do not support concurrent. + */ +void synchronize_srcu_async(struct srcu_struct *ssp, atomic_t *count) +{ + init_rcu_head(&ssp->srcu_async.head); + atomic_inc(count); + ssp->srcu_async.count = count; + call_srcu(ssp, &ssp->srcu_async.head, wakeme_after_rcu_async); +} +EXPORT_SYMBOL_GPL(synchronize_srcu_async); + /* Lockdep diagnostics. */ void __init rcu_scheduler_starting(void) {
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 6d3ef700fb0e..b4045cc0d7f2 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c@@ -1014,6 +1014,32 @@ void synchronize_srcu(struct srcu_struct *ssp) } EXPORT_SYMBOL_GPL(synchronize_srcu); +/* + * synchronize_srcu_async - do not wait until prior SRCU read-side + * critical-section completion, used for sync lots of SRCUs together + * to reduce wait time. + * Caution: do not support concurrent. + */ +void synchronize_srcu_async(struct srcu_struct *ssp, atomic_t *count) +{ + RCU_LOCKDEP_WARN(lock_is_held(&ssp->dep_map) || + lock_is_held(&rcu_bh_lock_map) || + lock_is_held(&rcu_lock_map) || + lock_is_held(&rcu_sched_lock_map), + "Illegal synchronize_srcu() in same-type SRCU (or in RCU) read-side critical section"); + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) + return; + + might_sleep(); + check_init_srcu_struct(ssp); + init_rcu_head(&ssp->srcu_async.head); + + atomic_inc(count); + ssp->srcu_async.count = count; + call_srcu(ssp, &ssp->srcu_async.head, wakeme_after_rcu_async); +} +EXPORT_SYMBOL_GPL(synchronize_srcu_async); + /* * Callback function for srcu_barrier() use. */
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index 84843adfd939..6433ae9c54cf 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c@@ -377,6 +377,17 @@ void wakeme_after_rcu(struct rcu_head *head) } EXPORT_SYMBOL_GPL(wakeme_after_rcu); +void wakeme_after_rcu_async(struct rcu_head *head) +{ + struct rcu_synchronize_async *rcu_asnc; + + rcu_asnc = container_of(head, struct rcu_synchronize_async, head); + + atomic_dec(rcu_asnc->count); + destroy_rcu_head_on_stack(&rcu_asnc->head); +} +EXPORT_SYMBOL_GPL(wakeme_after_rcu_async); + void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array, struct rcu_synchronize *rs_array) {
--
2.16.4