freelist_lock, which protects slow path allocations, is currently stored
in struct io_zcrx_area. Once we add support for multiple queues, we'll
need a lock in the zcrx ctx, move it there.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
io_uring/zcrx.c | 14 +++++++-------
io_uring/zcrx.h | 2 +-
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index be3560739bbc..3cf5e456990a 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -542,7 +542,6 @@ static int __zcrx_create_area(struct io_zcrx_ifq *ifq,
/* we're only supporting one area per ifq for now */
area->area_id = 0;
area_reg->rq_area_token = zcrx_area_id_to_token(area->area_id);
- spin_lock_init(&area->freelist_lock);
ret = io_zcrx_append_area(ifq, area);
if (!ret)
@@ -573,6 +572,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)
ifq->if_rxq = -1;
spin_lock_init(&ifq->ctx_lock);
spin_lock_init(&ifq->rq.lock);
+ spin_lock_init(&ifq->alloc_lock);
mutex_init(&ifq->pp_lock);
refcount_set(&ifq->refs, 1);
refcount_set(&ifq->user_refs, 1);
@@ -647,8 +647,9 @@ static void io_put_zcrx_ifq(struct io_zcrx_ifq *ifq)
static void io_zcrx_return_niov_freelist(struct net_iov *niov)
{
struct io_zcrx_area *area = io_zcrx_iov_to_area(niov);
+ struct io_zcrx_ifq *ifq = area->ifq;
- guard(spinlock_bh)(&area->freelist_lock);
+ guard(spinlock_bh)(&ifq->alloc_lock);
if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs))
return;
area->freelist[area->free_count++] = net_iov_idx(niov);@@ -658,7 +659,7 @@ static struct net_iov *zcrx_get_free_niov(struct io_zcrx_area *area)
{
unsigned niov_idx;
- lockdep_assert_held(&area->freelist_lock);
+ lockdep_assert_held(&area->ifq->alloc_lock);
if (unlikely(!area->free_count))
return NULL;@@ -1251,7 +1252,7 @@ static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *if
struct io_zcrx_area *area = ifq->area;
unsigned allocated = 0;
- guard(spinlock_bh)(&area->freelist_lock);
+ guard(spinlock_bh)(&ifq->alloc_lock);
for (allocated = 0; allocated < to_alloc; allocated++) {
struct net_iov *niov = zcrx_get_free_niov(area);@@ -1564,14 +1565,13 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq)
{
- struct io_zcrx_area *area = ifq->area;
struct net_iov *niov = NULL;
if (!ifq->kern_readable)
return NULL;
- scoped_guard(spinlock_bh, &area->freelist_lock)
- niov = zcrx_get_free_niov(area);
+ scoped_guard(spinlock_bh, &ifq->alloc_lock)
+ niov = zcrx_get_free_niov(ifq->area);
if (niov)
page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 4f8428db4aa4..18de02716453 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -37,7 +37,6 @@ struct io_zcrx_area {
u16 area_id;
/* freelist */
- spinlock_t freelist_lock ____cacheline_aligned_in_smp;
u32 free_count;
u32 *freelist;
@@ -66,6 +65,7 @@ struct io_zcrx_ifq {
bool kern_readable;
struct zcrx_rq rq ____cacheline_aligned_in_smp;
+ spinlock_t alloc_lock ____cacheline_aligned_in_smp;
u32 if_rxq;
struct device *dev;--
2.54.0