--- v19
+++ v9
@@ -1,124 +1,179 @@
-Introduce xdp_update_skb_shared_info routine to update frags array
-metadata in skb_shared_info data structure converting to a skb from
-a xdp_buff or xdp_frame.
-According to the current skb_shared_info architecture in
-xdp_frame/xdp_buff and to the xdp multi-buff support, there is
-no need to run skb_add_rx_frag() and reset frags array converting the buffer
-to a skb since the frag array will be in the same position for xdp_buff/xdp_frame
-and for the skb, we just need to update memory metadata.
-Introduce XDP_FLAGS_PF_MEMALLOC flag in xdp_buff_flags in order to mark
-the xdp_buff or xdp_frame as under memory-pressure if pages of the frags array
-are under memory pressure. Doing so we can avoid looping over all fragments in
-xdp_update_skb_shared_info routine. The driver is expected to set the
-flag constructing the xdp_buffer using xdp_buff_set_frag_pfmemalloc
-utility routine.
-Rely on xdp_update_skb_shared_info in __xdp_build_skb_from_frame routine
-converting the multi-buff xdp_frame to a skb after performing a XDP_REDIRECT.
+Introduce the capability to map non-linear xdp buffer running
+mvneta_xdp_submit_frame() for XDP_TX and XDP_REDIRECT
-Acked-by: John Fastabend <john.fastabend@gmail.com>
-Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
---
- include/net/xdp.h | 33 ++++++++++++++++++++++++++++++++-
- net/core/xdp.c | 12 ++++++++++++
- 2 files changed, 44 insertions(+), 1 deletion(-)
+ drivers/net/ethernet/marvell/mvneta.c | 112 +++++++++++++++++---------
+ 1 file changed, 76 insertions(+), 36 deletions(-)
-diff --git a/include/net/xdp.h b/include/net/xdp.h
-index 4ec7bdf0d937..e594016eb193 100644
---- a/include/net/xdp.h
-+++ b/include/net/xdp.h
-@@ -67,7 +67,10 @@ struct xdp_txq_info {
- };
+diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
+index f8993f7488b9..ab57be38ba03 100644
+--- a/drivers/net/ethernet/marvell/mvneta.c
++++ b/drivers/net/ethernet/marvell/mvneta.c
+@@ -1860,8 +1860,8 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp,
+ bytes_compl += buf->skb->len;
+ pkts_compl++;
+ dev_kfree_skb_any(buf->skb);
+- } else if (buf->type == MVNETA_TYPE_XDP_TX ||
+- buf->type == MVNETA_TYPE_XDP_NDO) {
++ } else if ((buf->type == MVNETA_TYPE_XDP_TX ||
++ buf->type == MVNETA_TYPE_XDP_NDO) && buf->xdpf) {
+ if (napi && buf->type == MVNETA_TYPE_XDP_TX)
+ xdp_return_frame_rx_napi(buf->xdpf);
+ else
+@@ -2055,47 +2055,87 @@ mvneta_xdp_put_buff(struct mvneta_port *pp, struct mvneta_rx_queue *rxq,
- enum xdp_buff_flags {
-- XDP_FLAGS_MULTI_BUFF = BIT(0), /* non-linear xdp buff */
-+ XDP_FLAGS_MULTI_BUFF = BIT(0), /* non-linear xdp buff */
-+ XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp multi-buff paged memory
-+ * is under pressure
-+ */
- };
+ static int
+ mvneta_xdp_submit_frame(struct mvneta_port *pp, struct mvneta_tx_queue *txq,
+- struct xdp_frame *xdpf, bool dma_map)
++ struct xdp_frame *xdpf, int *nxmit_byte, bool dma_map)
+ {
+- struct mvneta_tx_desc *tx_desc;
+- struct mvneta_tx_buf *buf;
+- dma_addr_t dma_addr;
++ struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
++ struct device *dev = pp->dev->dev.parent;
++ struct mvneta_tx_desc *tx_desc = NULL;
++ int i, num_frames = 1;
++ struct page *page;
++
++ if (unlikely(xdp_frame_is_mb(xdpf)))
++ num_frames += sinfo->nr_frags;
- struct xdp_buff {
-@@ -96,6 +99,16 @@ static __always_inline void xdp_buff_clear_mb(struct xdp_buff *xdp)
- xdp->flags &= ~XDP_FLAGS_MULTI_BUFF;
+- if (txq->count >= txq->tx_stop_threshold)
++ if (txq->count + num_frames >= txq->size)
+ return MVNETA_XDP_DROPPED;
+
+- tx_desc = mvneta_txq_next_desc_get(txq);
++ for (i = 0; i < num_frames; i++) {
++ struct mvneta_tx_buf *buf = &txq->buf[txq->txq_put_index];
++ skb_frag_t *frag = NULL;
++ int len = xdpf->len;
++ dma_addr_t dma_addr;
+
+- buf = &txq->buf[txq->txq_put_index];
+- if (dma_map) {
+- /* ndo_xdp_xmit */
+- dma_addr = dma_map_single(pp->dev->dev.parent, xdpf->data,
+- xdpf->len, DMA_TO_DEVICE);
+- if (dma_mapping_error(pp->dev->dev.parent, dma_addr)) {
+- mvneta_txq_desc_put(txq);
+- return MVNETA_XDP_DROPPED;
++ if (unlikely(i)) { /* paged area */
++ frag = &sinfo->frags[i - 1];
++ len = skb_frag_size(frag);
+ }
+- buf->type = MVNETA_TYPE_XDP_NDO;
+- } else {
+- struct page *page = virt_to_page(xdpf->data);
+
+- dma_addr = page_pool_get_dma_addr(page) +
+- sizeof(*xdpf) + xdpf->headroom;
+- dma_sync_single_for_device(pp->dev->dev.parent, dma_addr,
+- xdpf->len, DMA_BIDIRECTIONAL);
+- buf->type = MVNETA_TYPE_XDP_TX;
++ tx_desc = mvneta_txq_next_desc_get(txq);
++ if (dma_map) {
++ /* ndo_xdp_xmit */
++ void *data;
++
++ data = unlikely(frag) ? skb_frag_address(frag)
++ : xdpf->data;
++ dma_addr = dma_map_single(dev, data, len,
++ DMA_TO_DEVICE);
++ if (dma_mapping_error(dev, dma_addr)) {
++ mvneta_txq_desc_put(txq);
++ goto unmap;
++ }
++
++ buf->type = MVNETA_TYPE_XDP_NDO;
++ } else {
++ page = unlikely(frag) ? skb_frag_page(frag)
++ : virt_to_page(xdpf->data);
++ dma_addr = page_pool_get_dma_addr(page);
++ if (unlikely(frag))
++ dma_addr += skb_frag_off(frag);
++ else
++ dma_addr += sizeof(*xdpf) + xdpf->headroom;
++ dma_sync_single_for_device(dev, dma_addr, len,
++ DMA_BIDIRECTIONAL);
++ buf->type = MVNETA_TYPE_XDP_TX;
++ }
++ buf->xdpf = unlikely(i) ? NULL : xdpf;
++
++ tx_desc->command = unlikely(i) ? 0 : MVNETA_TXD_F_DESC;
++ tx_desc->buf_phys_addr = dma_addr;
++ tx_desc->data_size = len;
++ *nxmit_byte += len;
++
++ mvneta_txq_inc_put(txq);
+ }
+- buf->xdpf = xdpf;
+
+- tx_desc->command = MVNETA_TXD_FLZ_DESC;
+- tx_desc->buf_phys_addr = dma_addr;
+- tx_desc->data_size = xdpf->len;
++ /*last descriptor */
++ if (likely(tx_desc))
++ tx_desc->command |= MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD;
+
+- mvneta_txq_inc_put(txq);
+- txq->pending++;
+- txq->count++;
++ txq->pending += num_frames;
++ txq->count += num_frames;
+
+ return MVNETA_XDP_TX;
++
++unmap:
++ for (i--; i >= 0; i--) {
++ mvneta_txq_desc_put(txq);
++ tx_desc = txq->descs + txq->next_desc_to_proc;
++ dma_unmap_single(dev, tx_desc->buf_phys_addr,
++ tx_desc->data_size,
++ DMA_TO_DEVICE);
++ }
++
++ return MVNETA_XDP_DROPPED;
}
-+static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp)
-+{
-+ return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
-+}
-+
-+static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp)
-+{
-+ xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC;
-+}
-+
- static __always_inline void
- xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq)
- {
-@@ -151,6 +164,11 @@ static __always_inline bool xdp_frame_is_mb(struct xdp_frame *frame)
- return !!(frame->flags & XDP_FLAGS_MULTI_BUFF);
- }
+ static int
+@@ -2104,8 +2144,8 @@ mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp)
+ struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats);
+ struct mvneta_tx_queue *txq;
+ struct netdev_queue *nq;
++ int cpu, nxmit_byte = 0;
+ struct xdp_frame *xdpf;
+- int cpu;
+ u32 ret;
-+static __always_inline bool xdp_frame_is_frag_pfmemalloc(struct xdp_frame *frame)
-+{
-+ return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
-+}
-+
- #define XDP_BULK_QUEUE_SIZE 16
- struct xdp_frame_bulk {
- int count;
-@@ -186,6 +204,19 @@ static inline void xdp_scrub_frame(struct xdp_frame *frame)
- frame->dev_rx = NULL;
- }
+ xdpf = xdp_convert_buff_to_frame(xdp);
+@@ -2117,10 +2157,10 @@ mvneta_xdp_xmit_back(struct mvneta_port *pp, struct xdp_buff *xdp)
+ nq = netdev_get_tx_queue(pp->dev, txq->id);
-+static inline void
-+xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags,
-+ unsigned int size, unsigned int truesize,
-+ bool pfmemalloc)
-+{
-+ skb_shinfo(skb)->nr_frags = nr_frags;
-+
-+ skb->len += size;
-+ skb->data_len += size;
-+ skb->truesize += truesize;
-+ skb->pfmemalloc |= pfmemalloc;
-+}
-+
- /* Avoids inlining WARN macro in fast-path */
- void xdp_warn(const char *msg, const char *func, const int line);
- #define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)
-diff --git a/net/core/xdp.c b/net/core/xdp.c
-index 5ddc29f29bad..89183b2e3c07 100644
---- a/net/core/xdp.c
-+++ b/net/core/xdp.c
-@@ -529,8 +529,14 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
- struct sk_buff *skb,
- struct net_device *dev)
- {
-+ struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
- unsigned int headroom, frame_size;
- void *hard_start;
-+ u8 nr_frags;
-+
-+ /* xdp multi-buff frame */
-+ if (unlikely(xdp_frame_is_mb(xdpf)))
-+ nr_frags = sinfo->nr_frags;
+ __netif_tx_lock(nq, cpu);
+- ret = mvneta_xdp_submit_frame(pp, txq, xdpf, false);
++ ret = mvneta_xdp_submit_frame(pp, txq, xdpf, &nxmit_byte, false);
+ if (ret == MVNETA_XDP_TX) {
+ u64_stats_update_begin(&stats->syncp);
+- stats->es.ps.tx_bytes += xdpf->len;
++ stats->es.ps.tx_bytes += nxmit_byte;
+ stats->es.ps.tx_packets++;
+ stats->es.ps.xdp_tx++;
+ u64_stats_update_end(&stats->syncp);
+@@ -2159,11 +2199,11 @@ mvneta_xdp_xmit(struct net_device *dev, int num_frame,
- /* Part of headroom was reserved to xdpf */
- headroom = sizeof(*xdpf) + xdpf->headroom;
-@@ -550,6 +556,12 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
- if (xdpf->metasize)
- skb_metadata_set(skb, xdpf->metasize);
+ __netif_tx_lock(nq, cpu);
+ for (i = 0; i < num_frame; i++) {
+- ret = mvneta_xdp_submit_frame(pp, txq, frames[i], true);
++ ret = mvneta_xdp_submit_frame(pp, txq, frames[i], &nxmit_byte,
++ true);
+ if (ret != MVNETA_XDP_TX)
+ break;
-+ if (unlikely(xdp_frame_is_mb(xdpf)))
-+ xdp_update_skb_shared_info(skb, nr_frags,
-+ sinfo->xdp_frags_size,
-+ nr_frags * xdpf->frame_sz,
-+ xdp_frame_is_frag_pfmemalloc(xdpf));
-+
- /* Essential SKB info: protocol and skb->dev */
- skb->protocol = eth_type_trans(skb, dev);
+- nxmit_byte += frames[i]->len;
+ nxmit++;
+ }
--
2.31.1