net-next warning at kernel/softirq.c:159 local_bh_enable

Dave,

Shlomo saw too the ipv6 dst crash that was reported over
the list and also this warning when he rebased to net-next yesterday

Or.


[  285.551769] ------------[ cut here ]------------
[  285.552693] WARNING: at kernel/softirq.c:159 local_bh_enable+0x7a/0xa0()
[  285.552693] Hardware name: X7DWU
[  285.552693] Modules linked in: ib_ipoib ib_cm ib_sa mlx4_ib mlx4_en
mlx4_core ib_mad ib_core netconsole nfs lockd fscache auth_rpcgss nfs_acl
bnx2fc cnic 8021q uio garp fcoe libfcoe stp libfc llc scsi_transport_fc
scsi_tgt sunrpc binfmt_misc uinput joydev coretemp kvm_intel kvm microcode
pcspkr serio_raw i2c_i801 lpc_ich mfd_core i5400_edac edac_core i5k_amb
igb ptp pps_core ioatdma ixgbe dca mdio shpchp usb_storage floppy radeon
ttm drm_kms_helper drm i2c_algo_bit i2c_core [last unloaded:
scsi_wait_scan]

[  285.552693] Pid: 1662, comm: avahi-daemon Not tainted 3.5.0-rc5+ #2
[  285.552693] Call Trace:
[  285.552693]  [<ffffffff81058ebf>] warn_slowpath_common+0x7f/0xc0
[  285.552693]  [<ffffffff81058f1a>] warn_slowpath_null+0x1a/0x20
[  285.552693]  [<ffffffff810618ba>] local_bh_enable+0x7a/0xa0
[  285.552693]  [<ffffffff815294e0>] ipv4_neigh_lookup+0xc0/0x130
[  285.552693]  [<ffffffffa04f0966>] ipoib_mcast_send+0x1b6/0x480 [ib_ipoib]
[  285.552693]  [<ffffffff81093716>] ? update_sd_lb_stats+0x146/0x690
[  285.552693]  [<ffffffff814eb35b>] ? __alloc_skb+0x4b/0x170
[  285.552693]  [<ffffffffa04ec15d>] ipoib_path_lookup+0x11d/0x300 [ib_ipoib]
[  285.552693]  [<ffffffff814eb35b>] ? __alloc_skb+0x4b/0x170
[  285.552693]  [<ffffffff814e7718>] ? sock_alloc_send_pskb+0x1d8/0x380
[  285.552693]  [<ffffffffa04ec97d>] ipoib_start_xmit+0x1ad/0x430 [ib_ipoib]
[  285.552693]  [<ffffffff814fa64a>] dev_hard_start_xmit+0x24a/0x670
[  285.552693]  [<ffffffff8151790a>] sch_direct_xmit+0xfa/0x1d0
[  285.552693]  [<ffffffff814fabff>] dev_queue_xmit+0x18f/0x5e0
[  285.552693]  [<ffffffff81503667>] neigh_connected_output+0xc7/0x110
[  285.552693]  [<ffffffff81535c0d>] ip_finish_output+0x2ed/0x410
[  285.552693]  [<ffffffff81535ee9>] ip_mc_output+0x119/0x250
[  285.552693]  [<ffffffff81534fe9>] ip_local_out+0x29/0x30
[  285.552693]  [<ffffffff8153500b>] ip_send_skb+0x1b/0x50
[  285.552693]  [<ffffffff815598ad>] udp_send_skb+0x10d/0x3a0
[  285.552693]  [<ffffffff81534190>] ? ip_append_page+0x500/0x500
[  285.552693]  [<ffffffff8155ab9b>] udp_sendmsg+0x33b/0x970
[  285.552693]  [<ffffffff81080650>] ? update_rmtp+0x80/0x80
[  285.552693]  [<ffffffff81198011>] ? do_sys_poll+0x351/0x510
[  285.552693]  [<ffffffff81563a78>] inet_sendmsg+0x48/0xb0
[  285.552693]  [<ffffffff814e3398>] sock_sendmsg+0xf8/0x130
[  285.552693]  [<ffffffff81197330>] ? __pollwait+0xf0/0xf0
[  285.552693]  [<ffffffff81060707>] ? current_fs_time+0x27/0x30
[  285.552693]  [<ffffffff8119d3e1>] ? update_time+0x81/0xc0
[  285.552693]  [<ffffffff81197330>] ? __pollwait+0xf0/0xf0
[  285.552693]  [<ffffffff8119d469>] ? file_update_time+0x49/0xe0
[  285.552693]  [<ffffffff814e1e6e>] ? move_addr_to_kernel+0x4e/0x90
[  285.552693]  [<ffffffff814e1690>] ? copy_from_user+0x30/0x40
[  285.552693]  [<ffffffff814e4acd>] __sys_sendmsg+0x3fd/0x420
[  285.552693]  [<ffffffff81184aa2>] ? do_sync_write+0xe2/0x120
[  285.552693]  [<ffffffff811c1b05>] ? fsnotify+0x1d5/0x2f0
[  285.552693]  [<ffffffff814e4cf9>] sys_sendmsg+0x49/0x90
[  285.552693]  [<ffffffff81607da9>] system_call_fastpath+0x16/0x1b
[  285.552693] ---[ end trace cabd6aea7003ad6f ]---

From: Or Gerlitz <redacted>
Date: Fri, 6 Jul 2012 06:20:58 +0300

Shlomo saw too the ipv6 dst crash that was reported over
the list and also this warning when he rebased to net-next yesterday

Here is what I commited to fix this:

====================
[PATCH] ipoib: Need to do dst_neigh_lookup_skb() outside of priv->lock.

Otherwise local_bh_enable() complains.

Reported-by: Or Gerlitz <redacted>
Signed-off-by: David S. Miller <davem@davemloft.net>
---
 drivers/infiniband/ulp/ipoib/ipoib_multicast.c |   15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index fbb95ee..7cecb16 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c

@@ -658,9 +658,15 @@ static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
 void ipoib_mcast_send(struct net_device *dev, void *mgid, struct sk_buff *skb)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
+	struct dst_entry *dst = skb_dst(skb);
 	struct ipoib_mcast *mcast;
+	struct neighbour *n;
 	unsigned long flags;
 
+	n = NULL;
+	if (dst)
+		n = dst_neigh_lookup_skb(dst, skb);
+
 	spin_lock_irqsave(&priv->lock, flags);
 
 	if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)		||

@@ -715,12 +721,6 @@ void ipoib_mcast_send(struct net_device *dev, void *mgid, struct sk_buff *skb)
 
 out:
 	if (mcast && mcast->ah) {
-		struct dst_entry *dst = skb_dst(skb);
-		struct neighbour *n = NULL;
-
-		rcu_read_lock();
-		if (dst)
-			n = dst_neigh_lookup_skb(dst, skb);
 		if (n) {
 			if (!*to_ipoib_neigh(n)) {
 				struct ipoib_neigh *neigh;

@@ -735,13 +735,14 @@ out:
 			}
 			neigh_release(n);
 		}
-		rcu_read_unlock();
 		spin_unlock_irqrestore(&priv->lock, flags);
 		ipoib_send(dev, skb, mcast->ah, IB_MULTICAST_QPN);
 		return;
 	}
 
 unlock:
+	if (n)
+		neigh_release(n);
 	spin_unlock_irqrestore(&priv->lock, flags);
 }
 

-- 
1.7.10.4

FWIW - I've updated my net-next clone to the lastest and still
see this, even after the IPv6 related fix.

Or.


IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready
IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready
------------[ cut here ]------------
WARNING: at kernel/softirq.c:159 local_bh_enable+0x45/0xc2()
Hardware name: X7DWU
Modules linked in: ib_ipoib ib_cm ib_sa ib_uverbs netconsole nfs lockd auth_rpcgss nfs_acl autofs4 sunrpc fcoe libfcoe libfc scsi_transport_fc 8021q ipmi_devintf ipmi_si ipmi_msghandler ipv6 dm_mirror dm_region_hash dm_log uinput igb ptp pps_core mlx4_ib ib_mad ib_core mlx4_en mlx4_core sg kvm_intel kvm microcode pcspkr rng_core ioatdma dca dm_mod shpchp button sr_mod ext3 jbd usb_storage sd_mod ata_piix libata scsi_mod ehci_hcd uhci_hcd floppy [last unloaded: scsi_wait_scan]
Pid: 0, comm: swapper/0 Not tainted 3.5.0-rc5-12501-g1405080-dirty #86
Call Trace:
 <IRQ>  [<ffffffff81027f1d>] warn_slowpath_common+0x80/0x98
 [<ffffffffa0245266>] ? ip6_neigh_lookup+0x157/0x187 [ipv6]
 [<ffffffff81027f4a>] warn_slowpath_null+0x15/0x17
 [<ffffffff8102f56a>] local_bh_enable+0x45/0xc2
 [<ffffffffa0245266>] ip6_neigh_lookup+0x157/0x187 [ipv6]
 [<ffffffffa0245168>] ? ip6_neigh_lookup+0x59/0x187 [ipv6]
 [<ffffffffa030895d>] ipoib_mcast_send+0x375/0x48b [ib_ipoib]
 [<ffffffffa0308908>] ? ipoib_mcast_send+0x320/0x48b [ib_ipoib]
 [<ffffffff8102f5e2>] ? local_bh_enable+0xbd/0xc2
 [<ffffffffa030494f>] ipoib_path_lookup+0x2f3/0x331 [ib_ipoib]
 [<ffffffffa0245168>] ? ip6_neigh_lookup+0x59/0x187 [ipv6]
 [<ffffffffa0304b4d>] ipoib_start_xmit+0x1c0/0x54f [ib_ipoib]
 [<ffffffffa030498d>] ? ipoib_path_lookup+0x331/0x331 [ib_ipoib]
 [<ffffffff812c1be3>] dev_hard_start_xmit+0x449/0x60e
 [<ffffffff812c1804>] ? dev_hard_start_xmit+0x6a/0x60e
 [<ffffffff812d95b9>] sch_direct_xmit+0x72/0x201
 [<ffffffff812c2221>] dev_queue_xmit+0x479/0x70e
 [<ffffffff812c1da8>] ? dev_hard_start_xmit+0x60e/0x60e
 [<ffffffff812ca20e>] neigh_connected_output+0xb6/0xd4
 [<ffffffffa023c85b>] ip6_finish_output2+0x304/0x38e [ipv6]
 [<ffffffffa023c777>] ? ip6_finish_output2+0x220/0x38e [ipv6]
 [<ffffffffa0245057>] ? ip6_mtu+0x88/0xa1 [ipv6]
 [<ffffffffa023ca29>] ip6_finish_output+0x144/0x149 [ipv6]
 [<ffffffffa023cb41>] ip6_output+0x113/0x11c [ipv6]
 [<ffffffffa024c51f>] ndisc_send_skb+0x176/0x200 [ipv6]
 [<ffffffffa024c413>] ? ndisc_send_skb+0x6a/0x200 [ipv6]
 [<ffffffff81065350>] ? mark_held_locks+0xcd/0xf5
 [<ffffffffa024ce1a>] __ndisc_send+0x51/0x5d [ipv6]
 [<ffffffffa024e4a0>] ndisc_send_ns+0x75/0x82 [ipv6]
 [<ffffffffa02413ee>] addrconf_dad_timer+0x118/0x136 [ipv6]
 [<ffffffff8103547a>] run_timer_softirq+0x27a/0x38b
 [<ffffffff810353b5>] ? run_timer_softirq+0x1b5/0x38b
 [<ffffffffa02412d6>] ? addrconf_rs_timer+0xf2/0xf2 [ipv6]
 [<ffffffff8102f746>] __do_softirq+0xff/0x1de
 [<ffffffff81362dcc>] call_softirq+0x1c/0x26
 [<ffffffff81003090>] do_softirq+0x38/0x80
 [<ffffffff8102f41f>] irq_exit+0x4e/0x83
 [<ffffffff81019d2e>] smp_apic_timer_interrupt+0x86/0x94
 [<ffffffff8136255c>] apic_timer_interrupt+0x6c/0x80
 <EOI>  [<ffffffff810083ec>] ? mwait_idle+0x13c/0x208
 [<ffffffff810083e3>] ? mwait_idle+0x133/0x208
 [<ffffffff810088d1>] cpu_idle+0x6e/0xab
 [<ffffffff81343a23>] rest_init+0xc7/0xce
 [<ffffffff8134395c>] ? csum_partial_copy_generic+0x16c/0x16c
 [<ffffffff8167fbf3>] start_kernel+0x332/0x33f
 [<ffffffff8167f6f6>] ? kernel_init+0x19d/0x19d
 [<ffffffff8167f2b4>] x86_64_start_reservations+0xb8/0xbd
 [<ffffffff8167f3a6>] x86_64_start_kernel+0xed/0xf4
---[ end trace f71d7a59b646ab87 ]---

From: Or Gerlitz <redacted>
Date: Fri, 6 Jul 2012 07:17:33 +0300

FWIW - I've updated my net-next clone to the lastest and still
see this, even after the IPv6 related fix.

I sent you the infiniband patch necessary for this bug.

Here is what I commited to fix this:

====================
[PATCH] ipoib: Need to do dst_neigh_lookup_skb() outside of priv->lock.

Otherwise local_bh_enable() complains.

sorry, I missed your note, I just pulled from kernel.org and it
doesn't have the commit, will
be able to test that later.

Or.

On Fri, Jul 6, 2012 at 7:20 AM, David Miller [off-list ref] wrote:

I sent you the infiniband patch necessary for this bug.

OK, tested with the patch, all is well, thanks!

Or.