[PATCH net-next v3 2/3] ipv6: hash uncached routes by device
From: Chris J Arges <hidden>
Date: 2026-09-17 19:38:36
Also in:
linux-kselftest, lkml
Subsystem:
networking [general], networking [ipv4/ipv6], the rest · Maintainers:
"David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, David Ahern, Ido Schimmel, Linus Torvalds
rt6_uncached_list_flush_dev() currently walks every per-CPU uncached route list for each device being removed. Hash uncached routes by their inet6 device so ordinary device teardown only visits the matching bucket on each CPU. ip6_rt_get_dev_rcu() can return loopback or an L3 master while rt6i_idev still refers to the original interface. Key routes by rt6i_idev->dev when available and fall back to dst_dev(). Ordinary devices then require one bucket scan. Because loopback and L3 masters can instead be referenced by dst_dev(), scan all buckets when one of those devices is removed. This avoids growing struct rt6_info while filtering most unrelated routes from ordinary device teardown. The table has 64 buckets and costs approximately 1.5 KiB per possible CPU on x86-64. Signed-off-by: Chris J Arges <redacted> --- net/ipv6/route.c | 101 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 32 deletions(-)
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 7535b09068a0..cda81e91be65 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c@@ -40,6 +40,7 @@ #include <linux/seq_file.h> #include <linux/nsproxy.h> #include <linux/slab.h> +#include <linux/hash.h> #include <linux/jhash.h> #include <linux/siphash.h> #include <net/net_namespace.h>
@@ -133,11 +134,23 @@ struct uncached_list { struct list_head head; }; -static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt6_uncached_list); +#define RT6_UNCACHED_HASH_BITS 6 +#define RT6_UNCACHED_HASH_SIZE BIT(RT6_UNCACHED_HASH_BITS) + +struct rt6_uncached_table { + struct uncached_list buckets[RT6_UNCACHED_HASH_SIZE]; +}; + +static DEFINE_PER_CPU_ALIGNED(struct rt6_uncached_table, rt6_uncached_table); void rt6_uncached_list_add(struct rt6_info *rt) { - struct uncached_list *ul = raw_cpu_ptr(&rt6_uncached_list); + struct rt6_uncached_table *table = raw_cpu_ptr(&rt6_uncached_table); + struct uncached_list *ul; + struct net_device *dev; + + dev = rt->rt6i_idev ? rt->rt6i_idev->dev : dst_dev(&rt->dst); + ul = &table->buckets[hash_ptr(dev, RT6_UNCACHED_HASH_BITS)]; rt->dst.rt_uncached_list = ul;
@@ -157,40 +170,58 @@ void rt6_uncached_list_del(struct rt6_info *rt) } } +static void rt6_uncached_list_flush(struct uncached_list *ul, + struct net_device *dev) +{ + struct rt6_info *rt, *safe; + + if (list_empty(&ul->head)) + return; + + spin_lock_bh(&ul->lock); + list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { + struct net_device *rt_dev = dst_dev(&rt->dst); + struct inet6_dev *rt_idev = rt->rt6i_idev; + bool handled = false; + + if (rt_idev && rt_idev->dev == dev) { + rt->rt6i_idev = in6_dev_get(blackhole_netdev); + in6_dev_put(rt_idev); + handled = true; + } + + if (rt_dev == dev) { + rcu_assign_pointer(rt->dst.dev_rcu, blackhole_netdev); + netdev_ref_replace(rt_dev, blackhole_netdev, + &rt->dst.dev_tracker, GFP_ATOMIC); + handled = true; + } + if (handled) + list_del_init(&rt->dst.rt_uncached); + } + spin_unlock_bh(&ul->lock); +} + static void rt6_uncached_list_flush_dev(struct net_device *dev) { + bool scan_all = dev->flags & IFF_LOOPBACK || netif_is_l3_master(dev); int cpu; for_each_possible_cpu(cpu) { - struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); - struct rt6_info *rt, *safe; - - if (list_empty(&ul->head)) + struct rt6_uncached_table *table; + struct uncached_list *ul; + int bucket; + + table = per_cpu_ptr(&rt6_uncached_table, cpu); + if (!scan_all) { + ul = &table->buckets[hash_ptr(dev, + RT6_UNCACHED_HASH_BITS)]; + rt6_uncached_list_flush(ul, dev); continue; - - spin_lock_bh(&ul->lock); - list_for_each_entry_safe(rt, safe, &ul->head, dst.rt_uncached) { - struct inet6_dev *rt_idev = rt->rt6i_idev; - struct net_device *rt_dev = rt->dst.dev; - bool handled = false; - - if (rt_idev && rt_idev->dev == dev) { - rt->rt6i_idev = in6_dev_get(blackhole_netdev); - in6_dev_put(rt_idev); - handled = true; - } - - if (rt_dev == dev) { - rt->dst.dev = blackhole_netdev; - netdev_ref_replace(rt_dev, blackhole_netdev, - &rt->dst.dev_tracker, - GFP_ATOMIC); - handled = true; - } - if (handled) - list_del_init(&rt->dst.rt_uncached); } - spin_unlock_bh(&ul->lock); + + for (bucket = 0; bucket < RT6_UNCACHED_HASH_SIZE; bucket++) + rt6_uncached_list_flush(&table->buckets[bucket], dev); } }
@@ -6987,10 +7018,16 @@ int __init ip6_route_init(void) #endif for_each_possible_cpu(cpu) { - struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu); + struct rt6_uncached_table *table; + int bucket; + + table = per_cpu_ptr(&rt6_uncached_table, cpu); + for (bucket = 0; bucket < RT6_UNCACHED_HASH_SIZE; bucket++) { + struct uncached_list *ul = &table->buckets[bucket]; - INIT_LIST_HEAD(&ul->head); - spin_lock_init(&ul->lock); + INIT_LIST_HEAD(&ul->head); + spin_lock_init(&ul->lock); + } } out:
--
2.43.0