[PATCH v8 bpf-next 08/14] bpf: add multi-buff support to the bpf_xdp_adjust_tail() API
From: Lorenzo Bianconi <lorenzo@kernel.org>
Date: 2021-04-08 12:52:03
Also in:
bpf
Subsystem:
bpf [general] (safe dynamic programs and tools), bpf [networking] (tcx & tc bpf, sock_addr), networking [general], the rest, xdp (express data path) · Maintainers:
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi, "David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, Linus Torvalds, David S. Miller, Jesper Dangaard Brouer, John Fastabend
From: Eelco Chaudron <echaudro@redhat.com> This change adds support for tail growing and shrinking for XDP multi-buff. Signed-off-by: Eelco Chaudron <echaudro@redhat.com> Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org> --- include/net/xdp.h | 5 ++++ net/core/filter.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+)
diff --git a/include/net/xdp.h b/include/net/xdp.h
index c8eb7cf4ebed..55751cf2badf 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h@@ -159,6 +159,11 @@ static inline void xdp_set_frag_size(skb_frag_t *frag, u32 size) frag->bv_len = size; } +static inline unsigned int xdp_get_frag_tailroom(const skb_frag_t *frag) +{ + return PAGE_SIZE - xdp_get_frag_size(frag) - xdp_get_frag_offset(frag); +} + struct xdp_frame { void *data; u16 len;
diff --git a/net/core/filter.c b/net/core/filter.c
index cae56d08a670..c4eb1392f88e 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c@@ -3855,11 +3855,74 @@ static const struct bpf_func_proto bpf_xdp_adjust_head_proto = { .arg2_type = ARG_ANYTHING, }; +static int bpf_xdp_mb_adjust_tail(struct xdp_buff *xdp, int offset) +{ + struct xdp_shared_info *xdp_sinfo = xdp_get_shared_info_from_buff(xdp); + + if (unlikely(xdp_sinfo->nr_frags == 0)) + return -EINVAL; + + if (offset >= 0) { + skb_frag_t *frag = &xdp_sinfo->frags[xdp_sinfo->nr_frags - 1]; + int size; + + if (unlikely(offset > xdp_get_frag_tailroom(frag))) + return -EINVAL; + + size = xdp_get_frag_size(frag); + memset(xdp_get_frag_address(frag) + size, 0, offset); + xdp_set_frag_size(frag, size + offset); + xdp_sinfo->data_length += offset; + } else { + int i, frags_to_free = 0; + + offset = abs(offset); + + if (unlikely(offset > ((int)(xdp->data_end - xdp->data) + + xdp_sinfo->data_length - + ETH_HLEN))) + return -EINVAL; + + for (i = xdp_sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) { + skb_frag_t *frag = &xdp_sinfo->frags[i]; + int size = xdp_get_frag_size(frag); + int shrink = min_t(int, offset, size); + + offset -= shrink; + if (likely(size - shrink > 0)) { + /* When updating the final fragment we have + * to adjust the data_length in line. + */ + xdp_sinfo->data_length -= shrink; + xdp_set_frag_size(frag, size - shrink); + break; + } + + /* When we free the fragments, + * xdp_return_frags_from_buff() will take care + * of updating the xdp share info data_length. + */ + frags_to_free++; + } + + if (unlikely(frags_to_free)) + xdp_return_num_frags_from_buff(xdp, frags_to_free); + + if (unlikely(offset > 0)) + xdp->data_end -= offset; + } + + return 0; +} + BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset) { void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */ void *data_end = xdp->data_end + offset; + if (unlikely(xdp->mb)) + return bpf_xdp_mb_adjust_tail(xdp, offset); + /* Notice that xdp_data_hard_end have reserved some tailroom */ if (unlikely(data_end > data_hard_end)) return -EINVAL;
--
2.30.2