Update skb_splice_bits() (spd_fill_page(), sock_spd_release(), and
skb_pipe_buf_ops) and skb_append_pagefrags() to acquire and release
references via netmem_inc_ref(), netmem_dec_ref(), and skb_frag_ref()
so page_pool RX fragments spliced into a pipe remain pinned in the pool
while in the pipe and pair with skb_append_pagefrags() and
skb_release_data() when spliced into a TX socket once netmem_inc_ref()
and netmem_dec_ref() handle page_pool netmems.
In tls_push_data(), fall back to copying when MSG_SPLICE_PAGES
encounters a page_pool page because kTLS device offload record->frags[]
tracks pages via raw struct page references rather than pp_ref_count
before handing them to tls_push_sg().
Caveats:
- Because pipe buffers created by skb_splice_bits() hold pp_ref_count
until released (including across tee()), user pipes holding
unconsumed spliced data keep those page_pool pages in-flight and
DMA-mapped in the pool (bounded by pipe capacity and RLIMIT_NOFILE
rather than sk_rcvbuf) until the pipe buffers are consumed or closed,
with page_pool_destroy() deferring pool release via its periodic
workqueue and page_pool_scrub() unmapping DMA.
Signed-off-by: Mina Almasry <redacted>
---
net/core/skbuff.c | 31 +++++++++++++++++++++++++++----
net/tls/tls_device.c | 6 ++++++
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 48f6764117243..19d4ce347c41b 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3114,9 +3114,32 @@ EXPORT_SYMBOL(skb_copy_bits);
*/
static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i)
{
- put_page(spd->pages[i]);
+ netmem_dec_ref(page_to_netmem(spd->pages[i]));
}
+static void skb_pipe_buf_release(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
+{
+ netmem_dec_ref(page_to_netmem(buf->page));
+}
+
+static bool skb_pipe_buf_get(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
+{
+ struct page *page = compound_head(buf->page);
+
+ if (WARN_ON_ONCE(page_ref_count(page) <= 0))
+ return false;
+
+ netmem_inc_ref(page_to_netmem(page));
+ return true;
+}
+
+static const struct pipe_buf_operations skb_pipe_buf_ops = {
+ .release = skb_pipe_buf_release,
+ .get = skb_pipe_buf_get,
+};
+
static struct page *linear_to_page(struct page *page, unsigned int *len,
unsigned int *offset,
struct sock *sk)@@ -3165,7 +3188,7 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page,
spd->partial[spd->nr_pages - 1].len += *len;
return false;
}
- get_page(page);
+ netmem_inc_ref(page_to_netmem(page));
spd->pages[spd->nr_pages] = page;
spd->partial[spd->nr_pages].len = *len;
spd->partial[spd->nr_pages].offset = offset;
@@ -3281,7 +3304,7 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset,
.pages = pages,
.partial = partial,
.nr_pages_max = MAX_SKB_FRAGS,
- .ops = &nosteal_pipe_buf_ops,
+ .ops = &skb_pipe_buf_ops,
.spd_release = sock_spd_release,
};
int ret = 0;
@@ -4692,8 +4715,8 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page,
skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size);
} else if (i < max_frags) {
skb_zcopy_downgrade_managed(skb);
- get_page(page);
skb_fill_page_desc_noacc(skb, i, page, offset, size);
+ skb_frag_ref(skb, i);
} else {
return -EMSGSIZE;
}diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 4cdfbb57ecff0..05abf8e23f78a 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -513,10 +513,16 @@ static int tls_push_data(struct sock *sk,
goto handle_error;
}
+ if (page_pool_page_is_pp(compound_head(*pages))) {
+ iov_iter_revert(iter, copy);
+ goto copy_data;
+ }
+
zc_pfrag.offset = off;
zc_pfrag.size = copy;
tls_append_frag(record, &zc_pfrag, copy);
} else if (copy) {
+copy_data:
copy = min_t(size_t, copy, pfrag->size - pfrag->offset);
rc = tls_device_copy_data(page_address(pfrag->page) +--
2.56.0.455.g6c5a503b38-goog