xp_alloc() and xsk_buff_set_size() reset xdp_buff::flags of every
buffer they hand out. In aligned mode, duplicate fill ring addresses
map to the same xskb, so the head of a frame still being gathered can
be handed out again and lose XDP_FLAGS_HAS_FRAGS. xsk_buff_add_frag()
rejects the duplicate and the frame is dropped, but xsk_buff_free()
leaves its frags on pool->xskb_list, where they are taken as frags of
the next frame.
Keep XDP_FLAGS_HAS_FRAGS on hand-out and clear it when the frags are
released, in xsk_buff_free() and xsk_rcv_zc(). The driver XDP_TX paths,
bpf_xdp_adjust_tail() and bpf_xdp_pull_data() already clear it, and
pool->heads starts zeroed. Other flags are still reset on hand-out, as
nothing clears them on release.
Since xsk_rcv_zc() now clears the flag, i40e must count Rx bytes before
running the XDP program, like ice and libeth.
Fixes: f7f6aa8e2438 ("xsk: make xsk_buff_pool responsible for clearing xdp_buff::flags")
Signed-off-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
---
drivers/net/ethernet/intel/i40e/i40e_xsk.c | 2 +-
include/net/xdp_sock_drv.h | 4 +++-
net/xdp/xsk.c | 1 +
net/xdp/xsk_buff_pool.c | 2 +-
4 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
index e6ae4c7a5cb3..57b0f520fbb2 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c
@@ -354,7 +354,6 @@ static void i40e_handle_xdp_result_zc(struct i40e_ring *rx_ring,
struct sk_buff *skb;
*rx_packets = 1;
- *rx_bytes = xdp_get_buff_len(xdp_buff);
if (likely(xdp_res == I40E_XDP_REDIR) || xdp_res == I40E_XDP_TX)
return;
@@ -492,6 +491,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget)
xsk_buff_to_i40e_ctx(first)->desc = rx_desc;
+ rx_bytes = xdp_get_buff_len(first);
xdp_res = i40e_run_xdp_zc(rx_ring, first, xdp_prog);
i40e_handle_xdp_result_zc(rx_ring, first, rx_desc, &rx_packets,
&rx_bytes, xdp_res, &failure);
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index 29965e369598..a40ad3f90f13 100644
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -153,6 +153,7 @@ static inline void xsk_buff_free(struct xdp_buff *xdp)
}
xdp_get_shared_info_from_buff(xdp)->nr_frags = 0;
+ xdp_buff_clear_frags_flag(xdp);
out:
xp_free(xskb);
}
@@ -225,7 +226,8 @@ static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
xdp->data_meta = xdp->data;
xdp->data_end = xdp->data + size;
- xdp->flags = 0;
+ /* Frags flag is cleared when the frags of a head are released */
+ xdp->flags &= XDP_FLAGS_HAS_FRAGS;
}
static inline dma_addr_t xsk_buff_raw_get_dma(struct xsk_buff_pool *pool,
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 33475b180ea6..d22be82533ae 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -241,6 +241,7 @@ static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
goto err;
}
+ xdp_buff_clear_frags_flag(xdp);
__xsk_rcv_zc_safe(xs, xskb, len, contd);
xskb_list = &xskb->pool->xskb_list;
list_for_each_entry_safe(pos, tmp, xskb_list, list_node) {diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index 9d2d94f1fb75..4318ec98199e 100644
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -597,7 +597,7 @@ struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool)
xskb->xdp.data = xskb->xdp.data_hard_start + XDP_PACKET_HEADROOM;
xskb->xdp.data_meta = xskb->xdp.data;
- xskb->xdp.flags = 0;
+ xskb->xdp.flags &= XDP_FLAGS_HAS_FRAGS;
if (pool->dev)
xp_dma_sync_for_device(pool, xskb->dma, pool->frame_len);
--
2.43.0