Re: [PATCH v3 net-next 00/23] net: add preliminary netdev refcount tracking

On Tue, Dec 7, 2021 at 11:52 AM Andrew Lunn [off-list ref] wrote:

quoted

I just sent the netns tracker series.

I will shortly send the remainder of netdev tracking patches.

Hi Eric

Thanks for sending the full set of patches.

I applied them on top of net-next. And i can still reproduce my
issue. This is without any changes of mine, just net-next plus your
two new patchsets.

It is not easy to reproduce. I needed to start/stop the GNS3
simulation around 8 times before i triggered it. I don't know how GNS3
does it tairdown. It could be there are active daemons running in the
name spaces as it removes the veth and tap devices? That might explain
why some of the leaks seem to be from TCP connection setup attempts?

Given i now have your full patchset, do you think these traces are
valid? Are they pointing at real leaks?

[ 1423.515246] unregister_netdevice: waiting for eth0 to become free. Usage count = 9
[ 1423.515747] leaked reference.
[ 1423.515755]  dst_alloc+0x7a/0x180
[ 1423.515765]  ip6_dst_alloc+0x27/0x90
[ 1423.515771]  ip6_pol_route+0x257/0x430
[ 1423.515777]  ip6_pol_route_output+0x19/0x20
[ 1423.515781]  fib6_rule_lookup+0x18b/0x270
[ 1423.515789]  ip6_route_output_flags_noref+0xaa/0x110
[ 1423.515793]  ip6_route_output_flags+0x32/0xa0
[ 1423.515797]  ip6_dst_lookup_tail.constprop.0+0x181/0x240
[ 1423.515803]  ip6_dst_lookup_flow+0x43/0xa0
[ 1423.515808]  inet6_csk_route_socket+0x166/0x200
[ 1423.515815]  inet6_csk_xmit+0x56/0x130
[ 1423.515818]  __tcp_transmit_skb+0x53b/0xc30
[ 1423.515825]  __tcp_send_ack.part.0+0xc6/0x1a0
[ 1423.515829]  tcp_send_ack+0x1c/0x20
[ 1423.515834]  __tcp_ack_snd_check+0x42/0x200
[ 1423.515838]  tcp_rcv_established+0x27a/0x6f0
[ 1423.515842] leaked reference.
[ 1423.515843]  dst_alloc+0x7a/0x180
[ 1423.515848]  ip6_dst_alloc+0x27/0x90
[ 1423.515851]  ip6_pol_route+0x257/0x430
[ 1423.515855]  ip6_pol_route_output+0x19/0x20
[ 1423.515860]  fib6_rule_lookup+0x18b/0x270
[ 1423.515865]  ip6_route_output_flags_noref+0xaa/0x110
[ 1423.515869]  ip6_route_output_flags+0x32/0xa0
[ 1423.515872]  seg6_output_core+0x28d/0x320
[ 1423.515878]  seg6_output+0x33/0x120
[ 1423.515884]  lwtunnel_output+0x72/0xc0
[ 1423.515890]  ip6_local_out+0x61/0x70
[ 1423.515894]  ip6_send_skb+0x23/0x70
[ 1423.515898]  udp_v6_send_skb+0x207/0x460
[ 1423.515905]  udpv6_sendmsg+0xb13/0xdb0
[ 1423.515908]  inet6_sendmsg+0x65/0x70
[ 1423.515916]  sock_sendmsg+0x48/0x70
[ 1423.515920] leaked reference.
[ 1423.515922]  dst_alloc+0x7a/0x180
[ 1423.515926]  ip6_dst_alloc+0x27/0x90
[ 1423.515929]  ip6_pol_route+0x257/0x430
[ 1423.515933]  ip6_pol_route_output+0x19/0x20
[ 1423.515938]  fib6_rule_lookup+0x18b/0x270
[ 1423.515943]  ip6_route_output_flags_noref+0xaa/0x110
[ 1423.515946]  ip6_route_output_flags+0x32/0xa0
[ 1423.515949]  seg6_output_core+0x28d/0x320
[ 1423.515955]  seg6_output+0x33/0x120
[ 1423.515961]  lwtunnel_output+0x72/0xc0
[ 1423.515965]  ip6_local_out+0x61/0x70
[ 1423.515968]  ip6_send_skb+0x23/0x70
[ 1423.515973]  udp_v6_send_skb+0x207/0x460
[ 1423.515979]  udpv6_sendmsg+0xb13/0xdb0
[ 1423.515982]  inet6_sendmsg+0x65/0x70
[ 1423.515985]  sock_sendmsg+0x48/0x70
[ 1423.515988] leaked reference.
[ 1423.515990]  dst_alloc+0x7a/0x180
[ 1423.515993]  ip6_dst_alloc+0x27/0x90
[ 1423.515997]  ip6_pol_route+0x257/0x430
[ 1423.516001]  ip6_pol_route_output+0x19/0x20
[ 1423.516005]  fib6_rule_lookup+0x18b/0x270
[ 1423.516010]  ip6_route_output_flags_noref+0xaa/0x110
[ 1423.516014]  ip6_route_output_flags+0x32/0xa0
[ 1423.516017]  seg6_output_core+0x28d/0x320
[ 1423.516022]  seg6_output+0x33/0x120
[ 1423.516028]  lwtunnel_output+0x72/0xc0
[ 1423.516032]  ip6_local_out+0x61/0x70
[ 1423.516036]  ip6_send_skb+0x23/0x70
[ 1423.516040]  udp_v6_send_skb+0x207/0x460
[ 1423.516046]  udpv6_sendmsg+0xb13/0xdb0
[ 1423.516049]  inet6_sendmsg+0x65/0x70
[ 1423.516052]  sock_sendmsg+0x48/0x70
[ 1423.516055] leaked reference.
[ 1423.516057]  dst_alloc+0x7a/0x180
[ 1423.516061]  ip6_dst_alloc+0x27/0x90
[ 1423.516064]  ip6_pol_route+0x257/0x430
[ 1423.516068]  ip6_pol_route_output+0x19/0x20
[ 1423.516072]  fib6_rule_lookup+0x18b/0x270
[ 1423.516078]  ip6_route_output_flags_noref+0xaa/0x110
[ 1423.516081]  ip6_route_output_flags+0x32/0xa0
[ 1423.516084]  ip6_dst_lookup_tail.constprop.0+0x181/0x240
[ 1423.516088]  ip6_dst_lookup_flow+0x43/0xa0
[ 1423.516092]  inet6_csk_route_socket+0x166/0x200
[ 1423.516099]  inet6_csk_xmit+0x56/0x130
[ 1423.516102]  __tcp_transmit_skb+0x53b/0xc30
[ 1423.516106]  tcp_write_xmit+0x3d4/0x13f0
[ 1423.516110]  __tcp_push_pending_frames+0x37/0x100
[ 1423.516114]  tcp_push+0xd2/0x100
[ 1423.516122]  tcp_sendmsg_locked+0x2a7/0xdb0
[ 1423.516128] leaked reference.
[ 1423.516130]  dst_alloc+0x7a/0x180
[ 1423.516134]  ip6_dst_alloc+0x27/0x90
[ 1423.516137]  ip6_pol_route+0x257/0x430
[ 1423.516141]  ip6_pol_route_output+0x19/0x20
[ 1423.516146]  fib6_rule_lookup+0x18b/0x270
[ 1423.516151]  ip6_route_output_flags_noref+0xaa/0x110
[ 1423.516154]  ip6_route_output_flags+0x32/0xa0
[ 1423.516157]  ip6_dst_lookup_tail.constprop.0+0xde/0x240
[ 1423.516161]  ip6_dst_lookup_flow+0x43/0xa0
[ 1423.516166]  tcp_v6_connect+0x2a7/0x670
[ 1423.516171]  __inet_stream_connect+0xd1/0x3b0
[ 1423.516175]  inet_stream_connect+0x3b/0x60
[ 1423.516178]  __sys_connect_file+0x5f/0x70
[ 1423.516182]  __sys_connect+0xa2/0xd0
[ 1423.516186]  __x64_sys_connect+0x18/0x20
[ 1423.516190]  do_syscall_64+0x3b/0xc0
[ 1423.516197] leaked reference.
[ 1423.516198]  fib6_nh_init+0x30d/0x9c0
[ 1423.516203]  rtm_new_nexthop+0x68a/0x13a0
[ 1423.516208]  rtnetlink_rcv_msg+0x14f/0x380
[ 1423.516216]  netlink_rcv_skb+0x55/0x100
[ 1423.516222]  rtnetlink_rcv+0x15/0x20
[ 1423.516228]  netlink_unicast+0x230/0x340
[ 1423.516232]  netlink_sendmsg+0x252/0x4b0
[ 1423.516236]  sock_sendmsg+0x65/0x70
[ 1423.516240]  ____sys_sendmsg+0x24e/0x290
[ 1423.516243]  ___sys_sendmsg+0x81/0xc0
[ 1423.516247]  __sys_sendmsg+0x62/0xb0
[ 1423.516252]  __x64_sys_sendmsg+0x1d/0x20
[ 1423.516256]  do_syscall_64+0x3b/0xc0
[ 1423.516260]  entry_SYSCALL_64_after_hwframe+0x44/0xae
[ 1423.516266] leaked reference.
[ 1423.516267]  ipv6_add_dev+0x13e/0x4f0
[ 1423.516273]  addrconf_notify+0x2ca/0x950
[ 1423.516280]  raw_notifier_call_chain+0x49/0x60
[ 1423.516288]  call_netdevice_notifiers_info+0x50/0x90
[ 1423.516293]  __dev_change_net_namespace+0x30d/0x6c0
[ 1423.516299]  do_setlink+0xdc/0x10b0
[ 1423.516306]  __rtnl_newlink+0x608/0xa10
[ 1423.516312]  rtnl_newlink+0x49/0x70
[ 1423.516318]  rtnetlink_rcv_msg+0x14f/0x380
[ 1423.516323]  netlink_rcv_skb+0x55/0x100
[ 1423.516328]  rtnetlink_rcv+0x15/0x20
[ 1423.516333]  netlink_unicast+0x230/0x340
[ 1423.516337]  netlink_sendmsg+0x252/0x4b0
[ 1423.516345]  sock_sendmsg+0x65/0x70
[ 1423.516348]  ____sys_sendmsg+0x24e/0x290
[ 1423.516352]  ___sys_sendmsg+0x81/0xc0

Thanks for the report.

This all involves IPv6, and this might point to something I hinted
about last week.

Can you try :

Note that I am about to travel, and will be unable to respond to
emails for about 20 hours.
Thanks !

diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 8d834f901b48..043d270b65d6 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c

@@ -163,9 +163,6 @@ static void rt6_uncached_list_flush_dev(struct net

*net, struct net_device *dev)
        struct net_device *loopback_dev = net->loopback_dev;
        int cpu;

-       if (dev == loopback_dev)
-               return;
-
        for_each_possible_cpu(cpu) {
                struct uncached_list *ul = per_cpu_ptr(&rt6_uncached_list, cpu);
                struct rt6_info *rt;

@@ -175,12 +172,13 @@ static void rt6_uncached_list_flush_dev(struct

net *net, struct net_device *dev)
                        struct inet6_dev *rt_idev = rt->rt6i_idev;
                        struct net_device *rt_dev = rt->dst.dev;

-                       if (rt_idev->dev == dev) {
+                       if (rt_idev->dev == dev && dev != loopback_dev) {
                                rt->rt6i_idev = in6_dev_get(loopback_dev);
                                in6_dev_put(rt_idev);
                        }

                        if (rt_dev == dev) {
+                               WARN_ON_ONCE(dev == loopback_dev);
                                rt->dst.dev = blackhole_netdev;
                                dev_replace_track(rt_dev, blackhole_netdev,
                                                  &rt->dst.dev_tracker,