diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 3cf5e456990a..8de142856795 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -306,16 +306,14 @@ static int io_import_area(struct io_zcrx_ifq *ifq,
return io_import_umem(ifq, mem, area_reg);
}
-static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
+static void __io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area *area)
{
int i;
- if (!area)
- return;
+ lockdep_assert_held(&ifq->pp_lock);
- guard(mutex)(&ifq->pp_lock);
- if (!area->is_mapped)
+ if (!area || !area->is_mapped)
return;
area->is_mapped = false;
@@ -332,6 +330,23 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
}
}
+static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq,
+ struct io_zcrx_area *area)
+{
+ guard(mutex)(&ifq->pp_lock);
+ __io_zcrx_unmap_area(ifq, area);
+}
+
+static void io_zcrx_unmap_areas(struct io_zcrx_ifq *ifq)
+{
+ unsigned area_idx;
+
+ guard(mutex)(&ifq->pp_lock);
+
+ for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++)
+ __io_zcrx_unmap_area(ifq, ifq->areas[area_idx]);
+}
+
static void zcrx_sync_for_device(struct page_pool *pp, struct io_zcrx_ifq *zcrx,
netmem_ref *netmems, unsigned nr)
{@@ -458,13 +473,29 @@ static int io_zcrx_append_area(struct io_zcrx_ifq *ifq,
struct io_zcrx_area *area)
{
bool kern_readable = !area->mem.is_dmabuf;
+ struct io_zcrx_area **areas, **old_areas;
+ unsigned old_nr;
- if (WARN_ON_ONCE(ifq->area))
- return -EINVAL;
if (WARN_ON_ONCE(ifq->kern_readable != kern_readable))
return -EINVAL;
- ifq->area = area;
+ old_areas = ifq->areas;
+ old_nr = ifq->nr_areas;
+
+ areas = kmalloc_array(old_nr + 1, sizeof(areas[0]),
+ GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+ if (!areas)
+ return -ENOMEM;
+ if (old_areas)
+ memcpy(areas, old_areas, old_nr * sizeof(areas[0]));
+ areas[old_nr] = area;
+
+ scoped_guard(spinlock_bh, &ifq->rq.lock) {
+ guard(spinlock_bh)(&ifq->alloc_lock);
+ ifq->areas = areas;
+ ifq->nr_areas = old_nr + 1;
+ }
+ kfree(old_areas);
return 0;
}
@@ -609,7 +640,7 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)
if (ifq->if_rxq != -1)
netif_mp_close_rxq(netdev, ifq->if_rxq, &p);
- io_zcrx_unmap_area(ifq, ifq->area);
+ io_zcrx_unmap_areas(ifq);
netdev_unlock(netdev);
netdev_put(netdev, &netdev_tracker);
}
@@ -618,6 +649,8 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)
static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
{
+ int i;
+
if (WARN_ON_ONCE(ifq->if_rxq != -1))
return;
if (WARN_ON_ONCE(ifq->netdev != NULL))@@ -625,8 +658,8 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
if (WARN_ON_ONCE(ifq->master_ctx))
return;
- if (ifq->area)
- io_zcrx_free_area(ifq, ifq->area);
+ for (i = 0; i < ifq->nr_areas; i++)
+ io_zcrx_free_area(ifq, ifq->areas[i]);
if (ifq->mm_account)
mmdrop(ifq->mm_account);
if (ifq->dev)
@@ -635,6 +668,7 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)
io_free_rbuf_ring(ifq);
free_uid(ifq->user);
mutex_destroy(&ifq->pp_lock);
+ kfree(ifq->areas);
kfree(ifq);
}
@@ -680,14 +714,10 @@ static void io_zcrx_return_niov(struct net_iov *niov)
page_pool_put_unrefed_netmem(niov->desc.pp, netmem, -1, false);
}
-static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
+static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)
{
- struct io_zcrx_area *area = ifq->area;
int i;
- if (!area)
- return;
-
/* Reclaim back all buffers given to the user space. */
for (i = 0; i < area->nia.num_niovs; i++) {
struct net_iov *niov = &area->nia.niovs[i];@@ -701,6 +731,15 @@ static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
}
}
+static void io_zcrx_scrub(struct io_zcrx_ifq *ifq)
+{
+ int i;
+
+ guard(mutex)(&ifq->pp_lock);
+ for (i = 0; i < ifq->nr_areas; i++)
+ io_zcrx_scrub_area(ifq, ifq->areas[i]);
+}
+
static void zcrx_unregister_user(struct io_zcrx_ifq *ifq, struct io_ring_ctx *ctx)
{
scoped_guard(spinlock_bh, &ifq->ctx_lock) {@@ -1174,12 +1213,15 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
unsigned niov_idx, area_idx;
struct io_zcrx_area *area;
+ lockdep_assert_held(&ifq->rq.lock);
+
area_idx = off >> IORING_ZCRX_AREA_SHIFT;
niov_idx = (off & ~IORING_ZCRX_AREA_MASK) >> ifq->niov_shift;
- if (unlikely(rqe->__pad || area_idx))
+ if (unlikely(rqe->__pad || area_idx >= ifq->nr_areas))
return false;
- area = ifq->area;
+ area_idx = array_index_nospec(area_idx, ifq->nr_areas);
+ area = ifq->areas[area_idx];
if (unlikely(niov_idx >= area->nia.num_niovs))
return false;
@@ -1249,18 +1291,24 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
static unsigned io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq,
netmem_ref *netmems, unsigned to_alloc)
{
- struct io_zcrx_area *area = ifq->area;
+ unsigned area_idx = 0;
unsigned allocated = 0;
guard(spinlock_bh)(&ifq->alloc_lock);
- for (allocated = 0; allocated < to_alloc; allocated++) {
- struct net_iov *niov = zcrx_get_free_niov(area);
+ while (allocated < to_alloc) {
+ struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]);
+
+ if (!niov) {
+ area_idx++;
+ if (area_idx >= ifq->nr_areas)
+ break;
+ continue;
+ }
- if (!niov)
- break;
net_mp_niov_set_page_pool(pp, niov);
netmems[allocated] = net_iov_to_netmem(niov);
+ allocated++;
}
return allocated;
}@@ -1404,8 +1452,8 @@ static void io_pp_uninstall(void *mp_priv, struct netdev_rx_queue *rxq)
struct pp_memory_provider_params *p = &rxq->mp_params;
struct io_zcrx_ifq *ifq = mp_priv;
+ io_zcrx_unmap_areas(ifq);
io_zcrx_drop_netdev(ifq);
- io_zcrx_unmap_area(ifq, ifq->area);
p->mp_ops = NULL;
p->mp_priv = NULL;
@@ -1566,16 +1614,22 @@ static bool io_zcrx_queue_cqe(struct io_kiocb *req, struct net_iov *niov,
static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq)
{
struct net_iov *niov = NULL;
+ unsigned area_idx;
if (!ifq->kern_readable)
return NULL;
- scoped_guard(spinlock_bh, &ifq->alloc_lock)
- niov = zcrx_get_free_niov(ifq->area);
+ guard(spinlock_bh)(&ifq->alloc_lock);
+
+ for (area_idx = 0; area_idx < ifq->nr_areas; area_idx++) {
+ niov = zcrx_get_free_niov(ifq->areas[area_idx]);
+ if (niov) {
+ page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
+ return niov;
+ }
+ }
- if (niov)
- page_pool_fragment_netmem(net_iov_to_netmem(niov), 1);
- return niov;
+ return NULL;
}
struct io_copy_cache {diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h
index 18de02716453..d4a54b4e17fd 100644
--- a/io_uring/zcrx.h
+++ b/io_uring/zcrx.h
@@ -58,7 +58,10 @@ struct zcrx_rq {
};
struct io_zcrx_ifq {
- struct io_zcrx_area *area;
+ /* read-protected by any of: ->pp_lock, ->alloc_lock, ->rq.lock */
+ struct io_zcrx_area **areas;
+ unsigned nr_areas;
+
unsigned niov_shift;
struct user_struct *user;
struct mm_struct *mm_account;--
2.54.0