[RFC PATCH 10/11] ptq: Hook up receive side of Per Queue Threads
From: Tom Herbert <hidden>
Date: 2020-06-24 17:19:51
Subsystem:
networking drivers, networking [general], the rest · Maintainers:
Andrew Lunn, "David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, Linus Torvalds
Add code to set the queue in an rflow as opposed to just setting the CPU in an rps_dev_flow entry. set_rps_qid is the analogue for set_rps_cpu but for setting queues. In get_rps_cpu, a check is performed that identifier in the sock_flow_table refers to a queue; when it does call set_rps_qid after converting the global qid in the sock_flow_table to a device qid. In rps_record_sock_flow check is there is a per task receive queue for current (i.e. current->ptq_queues.rxq_id != NO_QUEUE). If there is a queue then set in sock_flow_table instead of setting the running CPU. Subsequently, the receive queue for the flow can be programmed by aRFS logic (ndo_rx_flow_steer). --- include/linux/netdevice.h | 28 ++++++++++++++++++++++++---- net/core/dev.c | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index ca163925211a..3b39be470720 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h@@ -731,12 +731,25 @@ static inline void rps_dev_flow_set_cpu(struct rps_dev_flow *dev_flow, u16 cpu) if (WARN_ON(cpu > RPS_MAX_CPU)) return; - /* Set the rflow target to the CPU atomically */ + /* Set the device flow target to the CPU atomically */ cpu_qid.use_qid = 0; cpu_qid.cpu = cpu; dev_flow->cpu_qid = cpu_qid; } +static inline void rps_dev_flow_set_qid(struct rps_dev_flow *dev_flow, u16 qid) +{ + struct rps_cpu_qid cpu_qid; + + if (WARN_ON(qid > RPS_MAX_QID)) + return; + + /* Set the device flow target to the CPU atomically */ + cpu_qid.use_qid = 1; + cpu_qid.qid = qid; + dev_flow->cpu_qid = cpu_qid; +} + /* * The rps_dev_flow_table structure contains a table of flow mappings. */
@@ -797,11 +810,18 @@ static inline void rps_record_sock_flow(struct rps_sock_flow_table *table, u32 hash) { if (table && hash) { - u32 val = hash & table->cpu_masks.hash_mask; unsigned int index = hash & table->mask; + u32 val; - /* We only give a hint, preemption can change CPU under us */ - val |= raw_smp_processor_id(); +#ifdef CONFIG_PER_THREAD_QUEUES + if (current->ptq_queues.rxq_id != NO_QUEUE) + val = RPS_SOCK_FLOW_USE_QID | + (hash & table->queue_masks.hash_mask) | + current->ptq_queues.rxq_id; + else +#endif + val = (hash & table->cpu_masks.hash_mask) | + raw_smp_processor_id(); if (table->ents[index] != val) table->ents[index] = val;
diff --git a/net/core/dev.c b/net/core/dev.c
index f4478c9b1c9c..1cad776e8847 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c@@ -4308,6 +4308,25 @@ set_rps_cpu(struct net_device *dev, struct sk_buff *skb, return rflow; } +static struct rps_dev_flow * +set_rps_qid(struct net_device *dev, struct sk_buff *skb, + struct rps_dev_flow *rflow, u16 qid) +{ + if (qid > RPS_MAX_QID) { + rps_dev_flow_clear(rflow); + return rflow; + } + +#ifdef CONFIG_RFS_ACCEL + /* Should we steer this flow to a different hardware queue? */ + if (skb_rx_queue_recorded(skb) && (dev->features & NETIF_F_NTUPLE) && + qid != skb_get_rx_queue(skb) && qid < dev->real_num_rx_queues) + set_arfs_queue(dev, skb, rflow, qid); +#endif + rps_dev_flow_set_qid(rflow, qid); + return rflow; +} + /* * get_rps_cpu is called from netif_receive_skb and returns the target * CPU from the RPS map of the receiving queue for a given skb.
@@ -4356,6 +4375,10 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, /* First check into global flow table if there is a match */ ident = sock_flow_table->ents[hash & sock_flow_table->mask]; + + if (ident == RPS_SOCK_FLOW_NO_IDENT) + goto try_rps; + comparator = ((ident & RPS_SOCK_FLOW_USE_QID) ? sock_flow_table->queue_masks.hash_mask : sock_flow_table->cpu_masks.hash_mask);
@@ -4372,8 +4395,21 @@ static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, * CPU. Proceed accordingly. */ if (ident & RPS_SOCK_FLOW_USE_QID) { + u16 dqid, gqid; + /* A queue identifier is in the sock_flow_table entry */ + gqid = ident & sock_flow_table->queue_masks.mask; + dqid = netdev_rx_gqid_to_dqid(dev, gqid); + + /* rflow has desired receive qid. Just set the qid in + * HW and return to use current CPU. Note that we + * don't consider OOO in this case. + */ + rflow = set_rps_qid(dev, skb, rflow, dqid); + + *rflowp = rflow; + /* Don't use aRFS to set CPU in this case, skip to * trying RPS */
--
2.25.1