diff --git a/include/net/netmem.h b/include/net/netmem.h
index f7dacc9e75fd..be6bc69c2f5a 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -116,6 +116,7 @@ struct net_iov {
};
struct net_iov_area *owner;
enum net_iov_type type;
+ atomic_t uref;
};
struct net_iov_area {diff --git a/include/net/sock.h b/include/net/sock.h
index 1e7f124871d2..e51ae567c121 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -350,7 +350,7 @@ struct sk_filter;
* @sk_scm_rights: flagged by SO_PASSRIGHTS to recv SCM_RIGHTS
* @sk_scm_unused: unused flags for scm_recv()
* @ns_tracker: tracker for netns reference
- * @sk_user_frags: xarray of pages the user is holding a reference on.
+ * @sk_devmem_binding: the devmem binding used by the socket
* @sk_owner: reference to the real owner of the socket that calls
* sock_lock_init_class_and_name().
*/
@@ -573,7 +573,7 @@ struct sock {
#endif
struct rcu_head sk_rcu;
netns_tracker ns_tracker;
- struct xarray sk_user_frags;
+ struct net_devmem_dmabuf_binding *sk_devmem_binding;
#if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES)
struct module *sk_owner;diff --git a/net/core/devmem.c b/net/core/devmem.c
index b4c570d4f37a..865d8dee539f 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -11,6 +11,7 @@
#include <linux/genalloc.h>
#include <linux/mm.h>
#include <linux/netdevice.h>
+#include <linux/skbuff_ref.h>
#include <linux/types.h>
#include <net/netdev_queues.h>
#include <net/netdev_rx_queue.h>
@@ -120,6 +121,7 @@ void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
struct netdev_rx_queue *rxq;
unsigned long xa_idx;
unsigned int rxq_idx;
+ int i;
xa_erase(&net_devmem_dmabuf_bindings, binding->id);
@@ -142,6 +144,20 @@ void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
__net_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
}
+ for (i = 0; i < binding->dmabuf->size / PAGE_SIZE; i++) {
+ struct net_iov *niov;
+ netmem_ref netmem;
+
+ niov = binding->vec[i];
+
+ if (!net_is_devmem_iov(niov))
+ continue;
+
+ netmem = net_iov_to_netmem(niov);
+ while (atomic_dec_and_test(&niov->uref))
+ WARN_ON_ONCE(!napi_pp_put_page(netmem));
+ }
+
net_devmem_dmabuf_binding_put(binding);
}
@@ -230,14 +246,12 @@ net_devmem_bind_dmabuf(struct net_device *dev,
goto err_detach;
}
- if (direction == DMA_TO_DEVICE) {
- binding->vec = kvmalloc_array(dmabuf->size / PAGE_SIZE,
- sizeof(struct net_iov *),
- GFP_KERNEL);
- if (!binding->vec) {
- err = -ENOMEM;
- goto err_unmap;
- }
+ binding->vec = kvmalloc_array(dmabuf->size / PAGE_SIZE,
+ sizeof(struct net_iov *),
+ GFP_KERNEL | __GFP_ZERO);
+ if (!binding->vec) {
+ err = -ENOMEM;
+ goto err_unmap;
}
/* For simplicity we expect to make PAGE_SIZE allocations, but the@@ -291,10 +305,10 @@ net_devmem_bind_dmabuf(struct net_device *dev,
niov = &owner->area.niovs[i];
niov->type = NET_IOV_DMABUF;
niov->owner = &owner->area;
+ atomic_set(&niov->uref, 0);
page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
net_devmem_get_dma_addr(niov));
- if (direction == DMA_TO_DEVICE)
- binding->vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
+ binding->vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
}
virtual += len;
diff --git a/net/core/devmem.h b/net/core/devmem.h
index 2ada54fb63d7..d4eb28d079bb 100644
--- a/net/core/devmem.h
+++ b/net/core/devmem.h
@@ -61,7 +61,7 @@ struct net_devmem_dmabuf_binding {
/* Array of net_iov pointers for this binding, sorted by virtual
* address. This array is convenient to map the virtual addresses to
- * net_iovs in the TX path.
+ * net_iovs.
*/
struct net_iov **vec;
diff --git a/net/core/sock.c b/net/core/sock.c
index 9a8290fcc35d..08d865ba04ea 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -87,6 +87,7 @@
#include <linux/unaligned.h>
#include <linux/capability.h>
+#include <linux/dma-buf.h>
#include <linux/errno.h>
#include <linux/errqueue.h>
#include <linux/types.h>
@@ -151,6 +152,7 @@
#include <uapi/linux/pidfd.h>
#include "dev.h"
+#include "devmem.h"
static DEFINE_MUTEX(proto_list_mutex);
static LIST_HEAD(proto_list);
@@ -1082,6 +1084,7 @@ sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen)
struct dmabuf_token *tokens;
int ret = 0, num_frags = 0;
netmem_ref netmems[16];
+ struct net_iov *niov;
if (!sk_is_tcp(sk))
return -EBADF;
@@ -1100,34 +1103,47 @@ sock_devmem_dontneed(struct sock *sk, sockptr_t optval, unsigned int optlen)
return -EFAULT;
}
- xa_lock_bh(&sk->sk_user_frags);
for (i = 0; i < num_tokens; i++) {
for (j = 0; j < tokens[i].token_count; j++) {
+ struct net_iov *niov;
+ unsigned int token;
+ netmem_ref netmem;
+
+ token = tokens[i].token_start + j;
+ if (token >= sk->sk_devmem_binding->dmabuf->size / PAGE_SIZE)
+ break;
+
if (++num_frags > MAX_DONTNEED_FRAGS)
goto frag_limit_reached;
-
- netmem_ref netmem = (__force netmem_ref)__xa_erase(
- &sk->sk_user_frags, tokens[i].token_start + j);
+ niov = sk->sk_devmem_binding->vec[token];
+ netmem = net_iov_to_netmem(niov);
if (!netmem || WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
continue;
+ if (atomic_dec_if_positive(&sk->sk_user_frags.urefs[token])
+ < 0)
+ continue;
+
netmems[netmem_num++] = netmem;
if (netmem_num == ARRAY_SIZE(netmems)) {
- xa_unlock_bh(&sk->sk_user_frags);
- for (k = 0; k < netmem_num; k++)
- WARN_ON_ONCE(!napi_pp_put_page(netmems[k]));
+ for (k = 0; k < netmem_num; k++) {
+ niov = netmem_to_net_iov(netmems[k]);
+ if (atomic_dec_and_test(&niov->uref))
+ WARN_ON_ONCE(!napi_pp_put_page(netmems[k]));
+ }
netmem_num = 0;
- xa_lock_bh(&sk->sk_user_frags);
}
ret++;
}
}
frag_limit_reached:
- xa_unlock_bh(&sk->sk_user_frags);
- for (k = 0; k < netmem_num; k++)
- WARN_ON_ONCE(!napi_pp_put_page(netmems[k]));
+ for (k = 0; k < netmem_num; k++) {
+ niov = netmem_to_net_iov(netmems[k]);
+ if (atomic_dec_and_test(&niov->uref))
+ WARN_ON_ONCE(!napi_pp_put_page(netmems[k]));
+ }
kvfree(tokens);
return ret;diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 40b774b4f587..813d73f23c87 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -261,6 +261,7 @@
#include <linux/memblock.h>
#include <linux/highmem.h>
#include <linux/cache.h>
+#include <linux/dma-buf.h>
#include <linux/err.h>
#include <linux/time.h>
#include <linux/slab.h>
@@ -475,7 +476,7 @@ void tcp_init_sock(struct sock *sk)
set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
sk_sockets_allocated_inc(sk);
- xa_init_flags(&sk->sk_user_frags, XA_FLAGS_ALLOC1);
+ sk->sk_devmem_binding = NULL;
}
EXPORT_IPV6_MOD(tcp_init_sock);
@@ -2386,68 +2387,6 @@ static int tcp_inq_hint(struct sock *sk)
return inq;
}
-/* batch __xa_alloc() calls and reduce xa_lock()/xa_unlock() overhead. */
-struct tcp_xa_pool {
- u8 max; /* max <= MAX_SKB_FRAGS */
- u8 idx; /* idx <= max */
- __u32 tokens[MAX_SKB_FRAGS];
- netmem_ref netmems[MAX_SKB_FRAGS];
-};
-
-static void tcp_xa_pool_commit_locked(struct sock *sk, struct tcp_xa_pool *p)
-{
- int i;
-
- /* Commit part that has been copied to user space. */
- for (i = 0; i < p->idx; i++)
- __xa_cmpxchg(&sk->sk_user_frags, p->tokens[i], XA_ZERO_ENTRY,
- (__force void *)p->netmems[i], GFP_KERNEL);
- /* Rollback what has been pre-allocated and is no longer needed. */
- for (; i < p->max; i++)
- __xa_erase(&sk->sk_user_frags, p->tokens[i]);
-
- p->max = 0;
- p->idx = 0;
-}
-
-static void tcp_xa_pool_commit(struct sock *sk, struct tcp_xa_pool *p)
-{
- if (!p->max)
- return;
-
- xa_lock_bh(&sk->sk_user_frags);
-
- tcp_xa_pool_commit_locked(sk, p);
-
- xa_unlock_bh(&sk->sk_user_frags);
-}
-
-static int tcp_xa_pool_refill(struct sock *sk, struct tcp_xa_pool *p,
- unsigned int max_frags)
-{
- int err, k;
-
- if (p->idx < p->max)
- return 0;
-
- xa_lock_bh(&sk->sk_user_frags);
-
- tcp_xa_pool_commit_locked(sk, p);
-
- for (k = 0; k < max_frags; k++) {
- err = __xa_alloc(&sk->sk_user_frags, &p->tokens[k],
- XA_ZERO_ENTRY, xa_limit_31b, GFP_KERNEL);
- if (err)
- break;
- }
-
- xa_unlock_bh(&sk->sk_user_frags);
-
- p->max = k;
- p->idx = 0;
- return k ? 0 : err;
-}
-
/* On error, returns the -errno. On success, returns number of bytes sent to the
* user. May not consume all of @remaining_len.
*/@@ -2456,14 +2395,11 @@ static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
int remaining_len)
{
struct dmabuf_cmsg dmabuf_cmsg = { 0 };
- struct tcp_xa_pool tcp_xa_pool;
unsigned int start;
int i, copy, n;
int sent = 0;
int err = 0;
- tcp_xa_pool.max = 0;
- tcp_xa_pool.idx = 0;
do {
start = skb_headlen(skb);
@@ -2510,8 +2446,12 @@ static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
*/
for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+ struct net_devmem_dmabuf_binding *binding;
struct net_iov *niov;
u64 frag_offset;
+ size_t size;
+ size_t len;
+ u32 token;
int end;
/* !skb_frags_readable() should indicate that ALL the@@ -2544,13 +2484,21 @@ static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
start;
dmabuf_cmsg.frag_offset = frag_offset;
dmabuf_cmsg.frag_size = copy;
- err = tcp_xa_pool_refill(sk, &tcp_xa_pool,
- skb_shinfo(skb)->nr_frags - i);
- if (err)
+
+ binding = net_devmem_iov_binding(niov);
+
+ if (!sk->sk_devmem_binding)
+ sk->sk_devmem_binding = binding;
+
+ if (sk->sk_devmem_binding != binding) {
+ err = -EFAULT;
goto out;
+ }
+
+ token = net_iov_virtual_addr(niov) >> PAGE_SHIFT;
+ dmabuf_cmsg.frag_token = token;
/* Will perform the exchange later */
- dmabuf_cmsg.frag_token = tcp_xa_pool.tokens[tcp_xa_pool.idx];
dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov);
offset += copy;@@ -2563,8 +2511,8 @@ static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
if (err)
goto out;
- atomic_long_inc(&niov->pp_ref_count);
- tcp_xa_pool.netmems[tcp_xa_pool.idx++] = skb_frag_netmem(frag);
+ if (atomic_inc_return(&niov->uref) == 1)
+ atomic_long_inc(&niov->pp_ref_count);
sent += copy;
@@ -2574,7 +2522,6 @@ static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
start = end;
}
- tcp_xa_pool_commit(sk, &tcp_xa_pool);
if (!remaining_len)
goto out;
@@ -2592,7 +2539,6 @@ static int tcp_recvmsg_dmabuf(struct sock *sk, const struct sk_buff *skb,
}
out:
- tcp_xa_pool_commit(sk, &tcp_xa_pool);
if (!sent)
sent = err;
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 1e58a8a9ff7a..0afa5461de33 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -87,6 +87,9 @@
#include <crypto/hash.h>
#include <linux/scatterlist.h>
+#include <linux/dma-buf.h>
+#include "../core/devmem.h"
+
#include <trace/events/tcp.h>
#ifdef CONFIG_TCP_MD5SIG
@@ -2526,25 +2529,10 @@ static void tcp_md5sig_info_free_rcu(struct rcu_head *head)
}
#endif
-static void tcp_release_user_frags(struct sock *sk)
-{
-#ifdef CONFIG_PAGE_POOL
- unsigned long index;
- void *netmem;
-
- xa_for_each(&sk->sk_user_frags, index, netmem)
- WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
-#endif
-}
-
void tcp_v4_destroy_sock(struct sock *sk)
{
struct tcp_sock *tp = tcp_sk(sk);
- tcp_release_user_frags(sk);
-
- xa_destroy(&sk->sk_user_frags);
-
trace_tcp_destroy_sock(sk);
tcp_clear_xmit_timers(sk);diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index d1c9e4088646..4e8ea73daab7 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -639,8 +639,6 @@ struct sock *tcp_create_openreq_child(const struct sock *sk,
__TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);
- xa_init_flags(&newsk->sk_user_frags, XA_FLAGS_ALLOC1);
-
return newsk;
}
EXPORT_SYMBOL(tcp_create_openreq_child);
--
2.47.3