On 11/04, Mina Almasry wrote:
quoted hunk ↗ jump to hunk
Enforce that all fragments in a nonlinear skb come from compatible
memory providers (and for devmem, the exact same binding):
- Add skb_has_net_iovs(skb), assuming skb->unreadable and net_iov-backed
frags are equivalent for now (with a comment noting the two can be
decoupled in the future if readable net_iovs are introduced).
- Add skb_net_iov_provider(), skb_mem_provider_compat(), and
skb_frags_can_coalesce() so multi-skb coalescing, shifting, and GRO
paths can share a single check across pp_recycle, skb_zcopy(), and
fragment memory providers:
- empty skbs (!len && !nr_frags && !skb_has_net_iovs()) are compatible
without mixing readable payload
- page vs net_iov is rejected via skb_has_net_iovs() (0 pointer chases)
- page vs page is allowed via !skb_has_net_iovs() (0 pointer chases)
- net_iov vs net_iov compares net_iov_owner(niov) for devmem and
niov->desc.pp for other net_iov providers (e.g., io_uring zcrx)
- Add net_devmem_skb_binding() and use it in
validate_xmit_unreadable_skb(), zerocopy_fill_skb_from_devmem(), and
tcp_sendmsg_locked() so frags from different devmem bindings or memory
providers cannot be mixed on TX.
- Use skb_frags_can_coalesce() across skb_try_coalesce(), skb_shift(),
and skb_gro_receive().
- Use skb_mem_provider_compat() in skb_gro_receive_list(),
tcp_skb_can_collapse(), and tcp_skb_can_collapse_rx().
- Reject converting linear head_frag payloads into page frags on
unreadable skbs in skb_try_coalesce() and skb_gro_receive().
- Reject unreadable skbs in tcp_can_coalesce_send_queue_head() since
tcp_clone_payload() does not propagate skb->unreadable and merges
frags via skb_frag_page() (which returns NULL for net_iovs).
This issue is theoretical and was found by manual code inspection and
LLM review; it was not triggered in production. Verified with the
net_core KUnit test suite; not tested on real hardware.
Fixes: 65249feb6b3d ("net: add support for skbs with unreadable frags")
Fixes: bd61848900bf ("net: devmem: Implement TX path")
Signed-off-by: Mina Almasry <redacted>
---
include/linux/skbuff.h | 74 ++++++++++++++++++++++++++++++++++++++++++
include/net/tcp.h | 5 +--
net/core/datagram.c | 4 +--
net/core/dev.c | 14 +++-----
net/core/devmem.h | 22 +++++++++++++
net/core/gro.c | 16 +++------
net/core/net_test.c | 39 ++++++++++++++++++----
net/core/skbuff.c | 28 +++++++---------
net/ipv4/tcp.c | 2 +-
net/ipv4/tcp_output.c | 3 ++
10 files changed, 160 insertions(+), 47 deletions(-)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 6787fcf6ac3e4..7723c24786c34 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1882,6 +1882,16 @@ static inline bool skb_frags_readable(const struct sk_buff *skb)
return !skb->unreadable;
}
+/* Return true if frags in this skb are backed by net_iovs.
+ * Assumes skb->unreadable and net_iov-backed frags are equivalent for now;
+ * the two can be decoupled in the future if readable net_iovs in skbs are
+ * introduced.
+ */
+static inline bool skb_has_net_iovs(const struct sk_buff *skb)
+{
+ return skb->unreadable;
+}
+
static inline void skb_mark_not_on_list(struct sk_buff *skb)
{
skb->next = NULL;@@ -3763,6 +3773,70 @@ static inline void *skb_frag_address(const skb_frag_t *frag)
return page_address(skb_frag_page(frag)) + skb_frag_off(frag);
}
+/* Return an opaque pointer identifying the memory provider instance of @skb's
+ * net_iov fragments:
+ * - For devmem (NET_IOV_DMABUF), net_iov_owner(niov) (&binding->area, 1:1 with
+ * the binding across RX queues and TX).
+ * - For other net_iov providers (e.g., io_uring zcrx), niov->desc.pp (shared
+ * across all areas of an io_zcrx_ifq).
+ */
+static inline void *skb_net_iov_provider(const struct sk_buff *skb)
+{
+ struct net_iov *niov;
+
[..]
+ if (!skb_has_net_iovs(skb) || !skb_shinfo(skb)->nr_frags)
+ return NULL;
(a few nits/questions)
Why !skb_has_net_iovs is not enough? (and if !nr_frags is required,
maybe fold it into skb_has_net_iovs?)
quoted hunk ↗ jump to hunk
+
+ niov = skb_frag_net_iov(&skb_shinfo(skb)->frags[0]);
+ if (!niov)
+ return NULL;
+
+ if (net_is_devmem_iov(niov))
+ return net_iov_owner(niov);
+
+ return niov->desc.pp;
+}
+
+/* Return true if the memory providers of @skb1 and @skb2 are compatible
+ * for coalescing. For any struct page-backed provider (the buddy allocator
+ * or a future provider handing out struct pages), coalescing is assumed
+ * feasible. For net_iov-backed providers, the provider instance pointer
+ * must match.
+ */
+static inline bool skb_mem_provider_compat(const struct sk_buff *skb1,
+ const struct sk_buff *skb2)
+{
+ void *prov1, *prov2;
+
[..]
quoted hunk ↗ jump to hunk
+ if ((!skb1->len && !skb_shinfo(skb1)->nr_frags &&
+ !skb_has_net_iovs(skb1)) ||
+ (!skb2->len && !skb_shinfo(skb2)->nr_frags &&
+ !skb_has_net_iovs(skb2)))
+ return true;
This deserves an explanation or a comment?
quoted hunk ↗ jump to hunk
+
+ if (skb_has_net_iovs(skb1) != skb_has_net_iovs(skb2))
+ return false;
+
[..]
+ if (!skb_has_net_iovs(skb1))
+ return true;
This also seems extra? The bottom prov1 == prov2 covers this?
quoted hunk ↗ jump to hunk
+
+ prov1 = skb_net_iov_provider(skb1);
+ prov2 = skb_net_iov_provider(skb2);
+ return prov1 && prov1 == prov2;
+}
+
+static inline bool skb_frags_can_coalesce(struct sk_buff *to,
+ struct sk_buff *from)
+{
+ if (to->pp_recycle != from->pp_recycle)
+ return false;
+
+ if (skb_zcopy(to) || skb_zcopy(from))
+ return false;
+
+ return skb_mem_provider_compat(to, from);
+}
+
/**
* skb_frag_address_safe - gets the address of the data contained in a paged fragment
* @frag: the paged fragment bufferdiff --git a/include/net/tcp.h b/include/net/tcp.h
index 3560adddf8215..07a222c0fe2d7 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1268,14 +1268,15 @@ static inline bool tcp_skb_can_collapse(const struct sk_buff *to,
mptcp_skb_can_collapse(to, from) &&
!skb_cmp_decrypted(to, from) &&
skb_pure_zcopy_same(to, from) &&
- skb_frags_readable(to) == skb_frags_readable(from));
+ skb_mem_provider_compat(to, from));
}
static inline bool tcp_skb_can_collapse_rx(const struct sk_buff *to,
const struct sk_buff *from)
{
return likely(mptcp_skb_can_collapse(to, from) &&
- !skb_cmp_decrypted(to, from));
+ !skb_cmp_decrypted(to, from) &&
+ skb_mem_provider_compat(to, from));
}
/* Events passed to congestion control interface */diff --git a/net/core/datagram.c b/net/core/datagram.c
index ecc30a6f83023..1e7db61e25a56 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -712,8 +712,8 @@ zerocopy_fill_skb_from_devmem(struct sk_buff *skb, struct iov_iter *from,
size_t virt_addr, size, off;
struct net_iov *niov;
- if (i && skb_frags_readable(skb))
- return -EFAULT;
+ if (skb->len && net_devmem_skb_binding(skb) != binding)
+ return -EMSGSIZE;
/* Devmem filling works by taking an IOVEC from the user where the
* iov_addrs are interpreted as an offset in bytes into the dma-buf to
diff --git a/net/core/dev.c b/net/core/dev.c
index 1aece2fe2429f..f241d8c650fcd 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4169,8 +4169,7 @@ static struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb,
struct net_device *dev)
{
- struct skb_shared_info *shinfo;
- struct net_iov *niov;
+ struct net_devmem_dmabuf_binding *binding;
if (likely(skb_frags_readable(skb) ||
dev->netmem_tx == NETMEM_TX_NO_DMA))@@ -4179,14 +4178,11 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb,
if (dev->netmem_tx == NETMEM_TX_NONE)
goto out_free;
- shinfo = skb_shinfo(skb);
-
/* All net_iovs in an unreadable skb share the same binding. */
- if (shinfo->nr_frags > 0) {
- niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0]));
- if (net_is_devmem_iov(niov) &&
- (net_devmem_iov_binding(niov)->direction != DMA_TO_DEVICE ||
- READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev))
+ if (skb_shinfo(skb)->nr_frags > 0) {
+ binding = net_devmem_skb_binding(skb);
+ if (!binding || binding->direction != DMA_TO_DEVICE ||
+ READ_ONCE(binding->dev) != dev)
goto out_free;
}
diff --git a/net/core/devmem.h b/net/core/devmem.h
index c4f476947dfaa..33817b9e7c007 100644
--- a/net/core/devmem.h
+++ b/net/core/devmem.h
@@ -10,6 +10,7 @@
#ifndef _NET_DEVMEM_H
#define _NET_DEVMEM_H
+#include <linux/skbuff.h>
#include <net/net_namespace.h>
#include <net/netmem.h>
#include <net/netdev_netlink.h>
@@ -117,6 +118,21 @@ net_devmem_iov_binding(const struct net_iov *niov)
struct net_devmem_dmabuf_binding, area);
}
+static inline struct net_devmem_dmabuf_binding *
+net_devmem_skb_binding(const struct sk_buff *skb)
+{
+ struct net_iov *niov;
+
[..]
+ if (!skb_has_net_iovs(skb) || !skb_shinfo(skb)->nr_frags)
+ return NULL;
Same question here about 'why !nr_frags' is needed. Can we simplify
(or cover this in !skb_has_net_iovs)