[PATCH net-next 1/3] net: dst_cache: implement RCU variants for dst_cache helpers
From: Marek Mietus <hidden>
Date: 2025-09-09 06:26:33
Subsystem:
networking [general], the rest · Maintainers:
"David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, Linus Torvalds
Implement RCU variants for existing dst_cache helpers interacting with dst_entry. The new helpers avoid referencing the dst_entry, sparing some unnecessary atomic operations. They should only be used in flows that are already guaranteed to be inside a RCU read-side critical context. Signed-off-by: Marek Mietus <redacted> --- include/net/dst_cache.h | 22 ++++++++++++ net/core/dst_cache.c | 78 +++++++++++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 10 deletions(-)
diff --git a/include/net/dst_cache.h b/include/net/dst_cache.h
index 1961699598e2..e94f6c15bed9 100644
--- a/include/net/dst_cache.h
+++ b/include/net/dst_cache.h@@ -32,6 +32,17 @@ struct dst_entry *dst_cache_get(struct dst_cache *dst_cache); */ struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr); +/** + * dst_cache_get_ip4_rcu - perform cache lookup and fetch ipv4 source + * address without taking a reference on the dst + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * Must be called with local BH disabled, and within an rcu read side + * critical section + */ +struct rtable *dst_cache_get_ip4_rcu(struct dst_cache *dst_cache, __be32 *saddr); + /** * dst_cache_set_ip4 - store the ipv4 dst into the cache * @dst_cache: the cache
@@ -43,6 +54,17 @@ struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr); void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, __be32 saddr); +/** + * dst_cache_steal_ip4 - store the ipv4 dst into the cache and steal its + * reference + * @dst_cache: the cache + * @dst: the entry to be cached whose reference will be stolen + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled + */ +void dst_cache_steal_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr); #if IS_ENABLED(CONFIG_IPV6) /**
diff --git a/net/core/dst_cache.c b/net/core/dst_cache.c
index 9ab4902324e1..f1e3992d8171 100644
--- a/net/core/dst_cache.c
+++ b/net/core/dst_cache.c@@ -25,20 +25,30 @@ struct dst_cache_pcpu { }; }; -static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, - struct dst_entry *dst, u32 cookie) +static void __dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, + struct dst_entry *dst, u32 cookie) { DEBUG_NET_WARN_ON_ONCE(!in_softirq()); dst_release(dst_cache->dst); - if (dst) - dst_hold(dst); dst_cache->cookie = cookie; dst_cache->dst = dst; } -static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, - struct dst_cache_pcpu *idst) +static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, + struct dst_entry *dst, u32 cookie) +{ + if (dst == dst_cache->dst && cookie == dst_cache->cookie) + return; + + if (dst) + dst_hold(dst); + + __dst_cache_per_cpu_dst_set(dst_cache, dst, cookie); +} + +static struct dst_entry *__dst_cache_per_cpu_get(struct dst_cache *dst_cache, + struct dst_cache_pcpu *idst) { struct dst_entry *dst;
@@ -47,14 +57,10 @@ static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, if (!dst) goto fail; - /* the cache already hold a dst reference; it can't go away */ - dst_hold(dst); - if (unlikely(!time_after(idst->refresh_ts, READ_ONCE(dst_cache->reset_ts)) || (READ_ONCE(dst->obsolete) && !dst->ops->check(dst, idst->cookie)))) { dst_cache_per_cpu_dst_set(idst, NULL, 0); - dst_release(dst); goto fail; } return dst;
@@ -64,6 +70,18 @@ static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, return NULL; } +static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, + struct dst_cache_pcpu *idst) +{ + struct dst_entry *dst; + + dst = __dst_cache_per_cpu_get(dst_cache, idst); + if (dst) + /* the cache already hold a dst reference; it can't go away */ + dst_hold(dst); + return dst; +} + struct dst_entry *dst_cache_get(struct dst_cache *dst_cache) { struct dst_entry *dst;
@@ -100,6 +118,28 @@ struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) } EXPORT_SYMBOL_GPL(dst_cache_get_ip4); +struct rtable *dst_cache_get_ip4_rcu(struct dst_cache *dst_cache, __be32 *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + local_lock_nested_bh(&dst_cache->cache->bh_lock); + idst = this_cpu_ptr(dst_cache->cache); + dst = __dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) { + local_unlock_nested_bh(&dst_cache->cache->bh_lock); + return NULL; + } + + *saddr = idst->in_saddr.s_addr; + local_unlock_nested_bh(&dst_cache->cache->bh_lock); + return dst_rtable(dst); +} +EXPORT_SYMBOL_GPL(dst_cache_get_ip4_rcu); + void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, __be32 saddr) {
@@ -116,6 +156,24 @@ void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, } EXPORT_SYMBOL_GPL(dst_cache_set_ip4); +void dst_cache_steal_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) { + dst_release(dst); + return; + } + + local_lock_nested_bh(&dst_cache->cache->bh_lock); + idst = this_cpu_ptr(dst_cache->cache); + __dst_cache_per_cpu_dst_set(idst, dst, 0); + idst->in_saddr.s_addr = saddr; + local_unlock_nested_bh(&dst_cache->cache->bh_lock); +} +EXPORT_SYMBOL_GPL(dst_cache_steal_ip4); + #if IS_ENABLED(CONFIG_IPV6) void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, const struct in6_addr *saddr)
--
2.51.0