diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index dd2c41ef6fbe..5aee4351dc4b 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -792,19 +792,32 @@ static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1);
}
-static int check_mergeable_len(struct net_device *dev, void *mrg_ctx,
- unsigned int len)
+/* Length of the mergeable buffer that add_recvbuf_mergeable() posted. */
+static unsigned int mergeable_ctx_to_buflen(void *mrg_ctx)
{
- unsigned int headroom, tailroom, room, truesize;
+ unsigned int headroom, tailroom, room;
- truesize = mergeable_ctx_to_truesize(mrg_ctx);
headroom = mergeable_ctx_to_headroom(mrg_ctx);
tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
room = SKB_DATA_ALIGN(headroom + tailroom);
- if (len > truesize - room) {
- pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
- dev->name, len, (unsigned long)(truesize - room));
+ return mergeable_ctx_to_truesize(mrg_ctx) - room;
+}
+
+/* Length of the small buffer that add_recvbuf_small() posts. */
+static unsigned int small_buflen(const struct virtnet_info *vi)
+{
+ return vi->hdr_len + GOOD_PACKET_LEN;
+}
+
+static int check_mergeable_len(struct net_device *dev, void *mrg_ctx,
+ unsigned int len)
+{
+ unsigned int buflen = mergeable_ctx_to_buflen(mrg_ctx);
+
+ if (len > buflen) {
+ pr_debug("%s: rx error: len %u exceeds buffer length %u\n",
+ dev->name, len, buflen);
DEV_STATS_INC(dev, rx_length_errors);
return -1;
}@@ -1797,15 +1810,15 @@ static struct page *xdp_linearize_page(struct net_device *dev,
p = virt_to_head_page(buf);
off = buf - page_address(p);
- if (rq->use_page_pool_dma)
- page_pool_dma_sync_for_cpu(rq->page_pool, p,
- off, buflen);
-
if (check_mergeable_len(dev, ctx, buflen)) {
page_pool_put_page(rq->page_pool, p, -1, true);
goto err_buf;
}
+ if (rq->use_page_pool_dma)
+ page_pool_dma_sync_for_cpu(rq->page_pool, p,
+ off, buflen);
+
/* guard against a misconfigured or uncooperative backend that
* is sending packet larger than the MTU.
*/@@ -2165,15 +2178,15 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
page = virt_to_head_page(buf);
offset = buf - page_address(page);
- if (rq->use_page_pool_dma)
- page_pool_dma_sync_for_cpu(rq->page_pool, page,
- offset, len);
-
if (check_mergeable_len(dev, ctx, len)) {
page_pool_put_page(rq->page_pool, page, -1, true);
goto err;
}
+ if (rq->use_page_pool_dma)
+ page_pool_dma_sync_for_cpu(rq->page_pool, page,
+ offset, len);
+
truesize = mergeable_ctx_to_truesize(ctx);
xdp_frags_truesz += truesize;
@@ -2446,15 +2459,15 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
u64_stats_add(&stats->bytes, len);
page = virt_to_head_page(buf);
+ if (check_mergeable_len(dev, ctx, len))
+ goto err_skb;
+
if (rq->use_page_pool_dma) {
offset = buf - page_address(page);
page_pool_dma_sync_for_cpu(rq->page_pool, page,
offset, len);
}
- if (check_mergeable_len(dev, ctx, len))
- goto err_skb;
-
truesize = mergeable_ctx_to_truesize(ctx);
curr_skb = virtnet_skb_append_frag(rq, head_skb, curr_skb, page,
buf, len, truesize);@@ -2574,12 +2587,22 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
/* Sync the memory before touching anything through buf,
* unless virtio core did it already.
+ *
+ * len comes from the device and is checked against the buffer
+ * only later, so never sync past the buffer posted to the device.
+ * The sync still covers the header, which receive_buf() and the
+ * mergeable drop path read. Big buffers never come from the page
+ * pool, so a buffer here is either mergeable or small.
*/
if (rq->use_page_pool_dma) {
struct page *page = virt_to_head_page(buf);
int offset = buf - page_address(page);
+ unsigned int buflen = vi->mergeable_rx_bufs ?
+ mergeable_ctx_to_buflen(ctx) :
+ small_buflen(vi);
- page_pool_dma_sync_for_cpu(rq->page_pool, page, offset, len);
+ page_pool_dma_sync_for_cpu(rq->page_pool, page, offset,
+ min(len, buflen));
}
/* About the flags below:@@ -2655,7 +2678,7 @@ static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
buf += VIRTNET_RX_PAD + xdp_headroom;
ctx = mergeable_len_to_ctx(alloc_len, xdp_headroom);
- err = virtnet_rq_submit(rq, buf, vi->hdr_len + GOOD_PACKET_LEN, ctx, gfp);
+ err = virtnet_rq_submit(rq, buf, small_buflen(vi), ctx, gfp);
if (err < 0)
page_pool_put_page(rq->page_pool, virt_to_head_page(buf), -1, false);