net: dsa: User per-cpu 64-bit statistics

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>
---
 net/dsa/dsa.c      | 10 +++++---
 net/dsa/dsa_priv.h |  2 +-
 net/dsa/slave.c    | 72 +++++++++++++++++++++++++++++++++++++++---------------
 3 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c
index 0ba842c08dd3..a91e520e735f 100644
--- a/net/dsa/dsa.c
+++ b/net/dsa/dsa.c

@@ -190,6 +190,7 @@ static int dsa_switch_rcv(struct sk_buff *skb, struct net_device *dev,
 {
 	struct dsa_switch_tree *dst = dev->dsa_ptr;
 	struct sk_buff *nskb = NULL;
+	struct pcpu_sw_netstats *s;
 	struct dsa_slave_priv *p;
 
 	if (unlikely(dst == NULL)) {

@@ -213,10 +214,11 @@ static int dsa_switch_rcv(struct sk_buff *skb, struct net_device *dev,
 	skb->pkt_type = PACKET_HOST;
 	skb->protocol = eth_type_trans(skb, skb->dev);
 
-	u64_stats_update_begin(&p->stats64.syncp);
-	p->stats64.rx_packets++;
-	p->stats64.rx_bytes += skb->len;
-	u64_stats_update_end(&p->stats64.syncp);
+	s = this_cpu_ptr(p->stats64);
+	u64_stats_update_begin(&s->syncp);
+	s->rx_packets++;
+	s->rx_bytes += skb->len;
+	u64_stats_update_end(&s->syncp);
 
 	netif_receive_skb(skb);
 

diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h
index 7aa0656296c2..306cff229def 100644
--- a/net/dsa/dsa_priv.h
+++ b/net/dsa/dsa_priv.h

@@ -77,7 +77,7 @@ struct dsa_slave_priv {
 	struct sk_buff *	(*xmit)(struct sk_buff *skb,
 					struct net_device *dev);
 
-	struct pcpu_sw_netstats	stats64;
+	struct pcpu_sw_netstats	*stats64;
 
 	/* DSA port data, such as switch, port index, etc. */
 	struct dsa_port		*dp;

diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index e196562035b1..605444ced06c 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c

@@ -352,12 +352,14 @@ static inline netdev_tx_t dsa_netpoll_send_skb(struct dsa_slave_priv *p,
 static netdev_tx_t dsa_slave_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct dsa_slave_priv *p = netdev_priv(dev);
+	struct pcpu_sw_netstats *s;
 	struct sk_buff *nskb;
 
-	u64_stats_update_begin(&p->stats64.syncp);
-	p->stats64.tx_packets++;
-	p->stats64.tx_bytes += skb->len;
-	u64_stats_update_end(&p->stats64.syncp);
+	s = this_cpu_ptr(p->stats64);
+	u64_stats_update_begin(&s->syncp);
+	s->tx_packets++;
+	s->tx_bytes += skb->len;
+	u64_stats_update_end(&s->syncp);
 
 	/* Transmit function may have to reallocate the original SKB,
 	 * in which case it must have freed it. Only free it here on error.

@@ -596,15 +598,26 @@ static void dsa_slave_get_ethtool_stats(struct net_device *dev,
 {
 	struct dsa_slave_priv *p = netdev_priv(dev);
 	struct dsa_switch *ds = p->dp->ds;
+	struct pcpu_sw_netstats *s;
 	unsigned int start;
-
-	do {
-		start = u64_stats_fetch_begin_irq(&p->stats64.syncp);
-		data[0] = p->stats64.tx_packets;
-		data[1] = p->stats64.tx_bytes;
-		data[2] = p->stats64.rx_packets;
-		data[3] = p->stats64.rx_bytes;
-	} while (u64_stats_fetch_retry_irq(&p->stats64.syncp, start));
+	int i;
+
+	for_each_possible_cpu(i) {
+		u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
+
+		s = per_cpu_ptr(p->stats64, i);
+		do {
+			start = u64_stats_fetch_begin_irq(&s->syncp);
+			tx_packets = s->tx_packets;
+			tx_bytes = s->tx_bytes;
+			rx_packets = s->rx_packets;
+			rx_bytes = s->rx_bytes;
+		} while (u64_stats_fetch_retry_irq(&s->syncp, start));
+		data[0] += tx_packets;
+		data[1] += tx_bytes;
+		data[2] += rx_packets;
+		data[3] += rx_bytes;
+	}
 	if (ds->ops->get_ethtool_stats)
 		ds->ops->get_ethtool_stats(ds, p->dp->index, data + 4);
 }

@@ -879,16 +892,28 @@ static void dsa_slave_get_stats64(struct net_device *dev,
 				  struct rtnl_link_stats64 *stats)
 {
 	struct dsa_slave_priv *p = netdev_priv(dev);
+	struct pcpu_sw_netstats *s;
 	unsigned int start;
+	int i;
 
 	netdev_stats_to_stats64(stats, &dev->stats);
-	do {
-		start = u64_stats_fetch_begin_irq(&p->stats64.syncp);
-		stats->tx_packets = p->stats64.tx_packets;
-		stats->tx_bytes = p->stats64.tx_bytes;
-		stats->rx_packets = p->stats64.rx_packets;
-		stats->rx_bytes = p->stats64.rx_bytes;
-	} while (u64_stats_fetch_retry_irq(&p->stats64.syncp, start));
+	for_each_possible_cpu(i) {
+		u64 tx_packets, tx_bytes, rx_packets, rx_bytes;
+
+		s = per_cpu_ptr(p->stats64, i);
+		do {
+			start = u64_stats_fetch_begin_irq(&s->syncp);
+			tx_packets = s->tx_packets;
+			tx_bytes = s->tx_bytes;
+			rx_packets = s->rx_packets;
+			rx_bytes = s->rx_bytes;
+		} while (u64_stats_fetch_retry_irq(&s->syncp, start));
+
+		stats->tx_packets += tx_packets;
+		stats->tx_bytes += tx_bytes;
+		stats->rx_packets += rx_packets;
+		stats->rx_bytes += rx_bytes;
+	}
 }
 
 void dsa_cpu_port_ethtool_init(struct ethtool_ops *ops)

@@ -1202,7 +1227,11 @@ int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
 	slave_dev->vlan_features = master->vlan_features;
 
 	p = netdev_priv(slave_dev);
-	u64_stats_init(&p->stats64.syncp);
+	p->stats64 = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+	if (!p->stats64) {
+		free_netdev(slave_dev);
+		return -ENOMEM;
+	}
 	p->dp = &ds->ports[port];
 	INIT_LIST_HEAD(&p->mall_tc_list);
 	p->xmit = dst->tag_ops->xmit;

@@ -1217,6 +1246,7 @@ int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
 		netdev_err(master, "error %d registering interface %s\n",
 			   ret, slave_dev->name);
 		ds->ports[port].netdev = NULL;
+		free_percpu(p->stats64);
 		free_netdev(slave_dev);
 		return ret;
 	}

@@ -1227,6 +1257,7 @@ int dsa_slave_create(struct dsa_switch *ds, struct device *parent,
 	if (ret) {
 		netdev_err(master, "error %d setting up slave phy\n", ret);
 		unregister_netdev(slave_dev);
+		free_percpu(p->stats64);
 		free_netdev(slave_dev);
 		return ret;
 	}

@@ -1249,6 +1280,7 @@ void dsa_slave_destroy(struct net_device *slave_dev)
 			of_phy_deregister_fixed_link(port_dn);
 	}
 	unregister_netdev(slave_dev);
+	free_percpu(p->stats64);
 	free_netdev(slave_dev);
 }
 

-- 
2.9.3

On Thu, 2017-08-03 at 21:33 -0700, Florian Fainelli wrote:

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

This seems to be a bug fix, it would be nice to get a proper tag like :

Fixes: f613ed665bb3 ("net: dsa: Add support for 64-bit statistics")

Problem here is that if multiple cpus can call dsa_switch_rcv() at the
same time, then u64_stats_update_begin() contract is not respected.

include/linux/u64_stats_sync.h states :

 * Usage :
 *
 * Stats producer (writer) should use following template granted it already got
 * an exclusive access to counters (a lock is already taken, or per cpu
 * data is used [in a non preemptable context])
 *
 *   spin_lock_bh(...) or other synchronization to get exclusive access
 *   ...
 *   u64_stats_update_begin(&stats->syncp);

On 08/03/2017 10:36 PM, Eric Dumazet wrote:

On Thu, 2017-08-03 at 21:33 -0700, Florian Fainelli wrote:

quoted

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

This seems to be a bug fix, it would be nice to get a proper tag like :

Fixes: f613ed665bb3 ("net: dsa: Add support for 64-bit statistics")

Right, should have been added, thanks!

Problem here is that if multiple cpus can call dsa_switch_rcv() at the
same time, then u64_stats_update_begin() contract is not respected.

This is really where I struggled understanding what is wrong in the
non-per CPU version, my understanding is that we have:

- writers for xmit executes in process context
- writers for receive executes from NAPI (from the DSA's master network
device through it's own NAPI doing netif_receive_skb -> netdev_uses_dsa
-> netif_receive_skb)

readers should all execute in process context. The test scenario that
led to a deadlock involved running iperf in the background, having a
while loop with both ifconfig and ethtool reading stats, and somehow
when iperf exited, either reader would just be locked. So I guess this
leaves us with the two writers not being mutually excluded then, right?

include/linux/u64_stats_sync.h states :

 * Usage :
 *
 * Stats producer (writer) should use following template granted it already got
 * an exclusive access to counters (a lock is already taken, or per cpu
 * data is used [in a non preemptable context])
 *
 *   spin_lock_bh(...) or other synchronization to get exclusive access
 *   ...
 *   u64_stats_update_begin(&stats->syncp);

-- 
Florian

On Fri, 2017-08-04 at 08:51 -0700, Florian Fainelli wrote:

On 08/03/2017 10:36 PM, Eric Dumazet wrote:

quoted

On Thu, 2017-08-03 at 21:33 -0700, Florian Fainelli wrote:

quoted

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

This seems to be a bug fix, it would be nice to get a proper tag like :

Fixes: f613ed665bb3 ("net: dsa: Add support for 64-bit statistics")

Right, should have been added, thanks!

quoted

Problem here is that if multiple cpus can call dsa_switch_rcv() at the
same time, then u64_stats_update_begin() contract is not respected.

This is really where I struggled understanding what is wrong in the
non-per CPU version, my understanding is that we have:

- writers for xmit executes in process context
- writers for receive executes from NAPI (from the DSA's master network
device through it's own NAPI doing netif_receive_skb -> netdev_uses_dsa
-> netif_receive_skb)

readers should all execute in process context. The test scenario that
led to a deadlock involved running iperf in the background, having a
while loop with both ifconfig and ethtool reading stats, and somehow
when iperf exited, either reader would just be locked. So I guess this
leaves us with the two writers not being mutually excluded then, right?

You could add a debug version of u64_stats_update_begin()

doing 

int ret = atomic_inc((atomic_t *)syncp);

BUG_ON(ret & 1);


And u64_stats_update_end()

int ret = atomic_inc((atomic_t *)syncp);

BUG_ON(!(ret & 1));


We probably could have a CONFIG_DEBUG_U64_STATS  that could be used on
64bit kernels as well...

On Fri, 2017-08-04 at 10:11 -0700, Eric Dumazet wrote:

You could add a debug version of u64_stats_update_begin()

doing 

int ret = atomic_inc((atomic_t *)syncp);

I meant atomic_inc_return() of course.

BUG_ON(ret & 1);


And u64_stats_update_end()

int ret = atomic_inc((atomic_t *)syncp);

BUG_ON(!(ret & 1));


We probably could have a CONFIG_DEBUG_U64_STATS  that could be used on
64bit kernels as well...

From: Florian Fainelli <f.fainelli@gmail.com>
Date: Thu,  3 Aug 2017 21:33:27 -0700

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

Signed-off-by: Florian Fainelli <f.fainelli@gmail.com>

Applied with appropriate Fixes: tag added.

Thanks.

On 08/04/2017 10:11 AM, Eric Dumazet wrote:

On Fri, 2017-08-04 at 08:51 -0700, Florian Fainelli wrote:

quoted

On 08/03/2017 10:36 PM, Eric Dumazet wrote:

quoted

On Thu, 2017-08-03 at 21:33 -0700, Florian Fainelli wrote:

quoted

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

This seems to be a bug fix, it would be nice to get a proper tag like :

Fixes: f613ed665bb3 ("net: dsa: Add support for 64-bit statistics")

Right, should have been added, thanks!

quoted

Problem here is that if multiple cpus can call dsa_switch_rcv() at the
same time, then u64_stats_update_begin() contract is not respected.

This is really where I struggled understanding what is wrong in the
non-per CPU version, my understanding is that we have:

- writers for xmit executes in process context
- writers for receive executes from NAPI (from the DSA's master network
device through it's own NAPI doing netif_receive_skb -> netdev_uses_dsa
-> netif_receive_skb)

readers should all execute in process context. The test scenario that
led to a deadlock involved running iperf in the background, having a
while loop with both ifconfig and ethtool reading stats, and somehow
when iperf exited, either reader would just be locked. So I guess this
leaves us with the two writers not being mutually excluded then, right?

You could add a debug version of u64_stats_update_begin()

doing 

int ret = atomic_inc((atomic_t *)syncp);

5 more quoted lines

so with your revised suggested patch:

static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        int ret = atomic_inc_return((atomic_t *)syncp);
        BUG_ON(ret & 1);
#endif
#if 0
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        write_seqcount_begin(&syncp->seq);
#endif
#endif
}

static inline void u64_stats_update_end(struct u64_stats_sync *syncp)
{
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        int ret = atomic_inc_return((atomic_t *)syncp);
        BUG_ON(!(ret & 1));
#endif
#if 0
#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
        write_seqcount_end(&syncp->seq);
#endif
#endif
}

and this makes us choke pretty early in IRQ accounting, did I get your
suggestion right?

[    0.015149] ------------[ cut here ]------------
[    0.020051] kernel BUG at ./include/linux/u64_stats_sync.h:82!
[    0.026221] Internal error: Oops - BUG: 0 [#1] SMP ARM
[    0.031661] Modules linked in:
[    0.034970] CPU: 0 PID: 0 Comm: swapper/0 Not tainted
4.13.0-rc5-01297-g7d3f0cd43fee-dirty #33
[    0.043990] Hardware name: Broadcom STB (Flattened Device Tree)
[    0.050237] task: c180a500 task.stack: c1800000

57 more lines

On 08/21/2017 04:23 PM, Florian Fainelli wrote:

On 08/04/2017 10:11 AM, Eric Dumazet wrote:

quoted

On Fri, 2017-08-04 at 08:51 -0700, Florian Fainelli wrote:

quoted

On 08/03/2017 10:36 PM, Eric Dumazet wrote:

quoted

On Thu, 2017-08-03 at 21:33 -0700, Florian Fainelli wrote:

quoted

During testing with a background iperf pushing 1Gbit/sec worth of
traffic and having both ifconfig and ethtool collect statistics, we
could see quite frequent deadlocks. Convert the often accessed DSA slave
network devices statistics to per-cpu 64-bit statistics to remove these
deadlocks and provide fast efficient statistics updates.

This seems to be a bug fix, it would be nice to get a proper tag like :

Fixes: f613ed665bb3 ("net: dsa: Add support for 64-bit statistics")

Right, should have been added, thanks!

quoted

Problem here is that if multiple cpus can call dsa_switch_rcv() at the
same time, then u64_stats_update_begin() contract is not respected.

This is really where I struggled understanding what is wrong in the
non-per CPU version, my understanding is that we have:

- writers for xmit executes in process context
- writers for receive executes from NAPI (from the DSA's master network
device through it's own NAPI doing netif_receive_skb -> netdev_uses_dsa
-> netif_receive_skb)

readers should all execute in process context. The test scenario that
led to a deadlock involved running iperf in the background, having a
while loop with both ifconfig and ethtool reading stats, and somehow
when iperf exited, either reader would just be locked. So I guess this
leaves us with the two writers not being mutually excluded then, right?

You could add a debug version of u64_stats_update_begin()

doing 

int ret = atomic_inc((atomic_t *)syncp);

37 more quoted lines

Well if we return 1 from atomic_inc_return() and the previous value was
zero, of course we are going to be bugging here. The idea behind the
patch I suppose is to make sure that we always get an odd number upon
u64_stats_update_begin()/entry, and an even number upon
u64_stats_update_end()/exit, right?

[    0.015149] ------------[ cut here ]------------
[    0.020051] kernel BUG at ./include/linux/u64_stats_sync.h:82!
[    0.026221] Internal error: Oops - BUG: 0 [#1] SMP ARM
[    0.031661] Modules linked in:
[    0.034970] CPU: 0 PID: 0 Comm: swapper/0 Not tainted
4.13.0-rc5-01297-g7d3f0cd43fee-dirty #33
[    0.043990] Hardware name: Broadcom STB (Flattened Device Tree)
[    0.050237] task: c180a500 task.stack: c1800000
[    0.055065] PC is at irqtime_account_delta+0xa4/0xa8
[    0.060322] LR is at 0x1
[    0.063057] pc : [<c0250504>]    lr : [<00000001>]    psr: 000001d3
[    0.069652] sp : c1801eec  ip : ee78b458  fp : c0e5ea48
[    0.075212] r10: c18b4b40  r9 : f0803000  r8 : ee00a800
[    0.080781] r7 : 00000001  r6 : c180a500  r5 : c1800000  r4 : 00000000
[    0.087680] r3 : 00000000  r2 : 0000ec8c  r1 : ee78b3c0  r0 : ee78b440
[    0.094546] Flags: nzcv  IRQs off  FIQs off  Mode SVC_32  ISA ARM
Segment user
[    0.102314] Control: 30c5387d  Table: 00003000  DAC: fffffffd
[    0.108414] Process swapper/0 (pid: 0, stack limit = 0xc1800210)
[    0.114791] Stack: (0xc1801eec to 0xc1802000)
[    0.119431] 1ee0:                            ee78b440 c1800000
c180a500 00000001 c02505c8
[    0.128079] 1f00: 00000004 ee00a800 ffffe000 00000000 00000000
c0227890 c17e6f20 c0278910
[    0.136665] 1f20: c185724c c18079a0 f080200c c1801f58 f0802000
c0201494 c0e00c18 20000053
[    0.145303] 1f40: ffffffff c1801f8c ffffffff c1800000 c18b4b40
c020d238 00000000 0000001f
[    0.153915] 1f60: 00040d00 00000000 efffc940 00000000 c18b4b40
c1807440 ffffffff 00000000
[    0.162571] 1f80: c18b4b40 c0e5ea48 00000004 c1801fa8 c0322fb0
c0e00c18 20000053 ffffffff
[    0.171226] 1fa0: c18b4b40 00000000 ffffffff ffffffff 00000000
c0e006c0 ffffffff 00000000
[    0.179890] 1fc0: 00000000 c1807448 c0e5ea48 00000000 00000000
c18b4dd4 c180745c c0e5ea44
[    0.188546] 1fe0: c180c0d0 00007000 420f00f3 00000000 00000000
00008090 00000000 00000000
[    0.197165] [<c0250504>] (irqtime_account_delta) from [<c02505c8>]

25 more quoted lines

-- 
Florian

On Mon, 2017-08-21 at 17:10 -0700, Florian Fainelli wrote:

Well if we return 1 from atomic_inc_return() and the previous value was
zero, of course we are going to be bugging here. The idea behind the
patch I suppose is to make sure that we always get an odd number upon
u64_stats_update_begin()/entry, and an even number upon
u64_stats_update_end()/exit, right?

Yes, this is right.