With large rx piages I often see >10 sequential RQEs referring to the
same niov. Instead of putting them one by one, count such RQEs during
parsing and batch refcounting for the niov.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 56 +++++++++++++++++++++++++++++++------------------
1 file changed, 36 insertions(+), 20 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 38557a9194ef..691206a7721b 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -358,16 +358,16 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov)
return &area->user_refs[net_iov_idx(niov)];
}
-static bool io_zcrx_put_niov_uref(struct net_iov *niov)
+static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs)
{
atomic_t *uref = io_get_user_counter(niov);
int old;
old = atomic_read(uref);
do {
- if (unlikely(old == 0))
+ if (unlikely(old < refs))
return false;
- } while (!atomic_try_cmpxchg(uref, &old, old - 1));
+ } while (!atomic_try_cmpxchg(uref, &old, old - refs));
return true;
}@@ -1163,6 +1163,22 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
return true;
}
+static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp,
+ unsigned refs)
+{
+ netmem_ref netmem = net_iov_to_netmem(niov);
+
+ if (!io_zcrx_put_niov_uref(niov, refs))
+ return false;
+ if (page_pool_unref_netmem(netmem, refs) != 0)
+ return false;
+ if (unlikely(niov->desc.pp != pp)) {
+ io_zcrx_return_niov(niov);
+ return false;
+ }
+ return true;
+}
+
static unsigned io_zcrx_ring_refill(struct page_pool *pp,
struct io_zcrx_ifq *ifq,
netmem_ref *netmems, unsigned to_alloc)@@ -1170,34 +1186,34 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
struct zcrx_rq *rq = &ifq->rq;
struct io_uring_zcrx_rqe *rqe;
struct zcrx_rq_iter it;
+ struct net_iov *niov = NULL;
+ unsigned niov_refs = 0;
unsigned allocated = 0;
guard(spinlock_bh)(&rq->lock);
zcrx_rq_iter_init(&it, rq);
- while (zcrx_rq_iter_next(&it, rq, &rqe)) {
- struct net_iov *niov;
- netmem_ref netmem;
+ while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) {
+ struct net_iov *next_niov;
- if (!io_parse_rqe(rqe, ifq, &niov))
- continue;
- if (!io_zcrx_put_niov_uref(niov))
+ if (!io_parse_rqe(rqe, ifq, &next_niov))
continue;
-
- netmem = net_iov_to_netmem(niov);
- if (!page_pool_unref_and_test(netmem))
- continue;
-
- if (unlikely(niov->desc.pp != pp)) {
- io_zcrx_return_niov(niov);
+ if (niov == next_niov) {
+ niov_refs++;
continue;
}
+ if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
+ netmems[allocated] = net_iov_to_netmem(niov);
+ allocated++;
+ }
+ niov = next_niov;
+ niov_refs = 1;
+ }
- netmems[allocated] = netmem;
+ if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
+ netmems[allocated] = net_iov_to_netmem(niov);
allocated++;
- if (allocated >= to_alloc)
- break;
}
smp_store_release(&rq->ring->head, rq->cached_head);@@ -1409,7 +1425,7 @@ static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr)
netmem_ref netmem = netmems[i];
struct net_iov *niov = netmem_to_net_iov(netmem);
- if (!io_zcrx_put_niov_uref(niov))
+ if (!io_zcrx_put_niov_uref(niov, 1))
continue;
if (!page_pool_unref_and_test(netmem))
continue;
--
2.54.0