Normally, there is just one io_uring instance using zcrx and all
receiving happens under its lock. In this case we can avoid grabbing the
ptr ring lock on the production side.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 6 +++++-
io_uring/zcrx.h | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 80a26b5798d3..3d5d5c9fd9a5 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -825,6 +825,7 @@ static int zcrx_export(struct io_ring_ctx *ctx, struct io_zcrx_ifq *ifq,
if (!mem_is_zero(ce, sizeof(*ce)))
return -EINVAL;
+ ifq->shared = true;
refcount_inc(&ifq->refs);
refcount_inc(&ifq->user_refs);
@@ -1974,7 +1975,10 @@ static int __zcrx_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
if (can_steal && !__ptr_ring_full(&ifq->skb_ring) &&
tcp_read_sock_steal_skb(desc, skb, args->sock->sk)) {
- ret = ptr_ring_produce(&ifq->skb_ring, skb);
+ if (ifq->shared)
+ ret = ptr_ring_produce(&ifq->skb_ring, skb);
+ else
+ ret = __ptr_ring_produce(&ifq->skb_ring, skb);
if (ret) {
zcrx_user_ref_frags(ifq, skb, first_frag, i);
__kfree_skb(skb);diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 7fc12e53c8a1..5ff4dabb0f68 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -71,6 +71,7 @@ struct io_zcrx_ifq {
spinlock_t alloc_lock ____cacheline_aligned_in_smp;
struct ptr_ring skb_ring;
+ bool shared;
u32 if_rxq;
struct device *dev;--
2.54.0