diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst
index 0f6088c4ab8bb872e7fc86f02479592e84c0247a..103328dc409e4485a3f989c95348c1bc74ab20c6 100644
--- a/Documentation/networking/net_cachelines/tcp_sock.rst
+++ b/Documentation/networking/net_cachelines/tcp_sock.rst
@@ -50,6 +50,7 @@ u8:1 tcp_usec_ts read_mostly read_m
u32 chrono_start read_write tcp_chrono_start/stop(tcp_write_xmit,tcp_cwnd_validate,tcp_send_syn_data)
u32[3] chrono_stat read_write tcp_chrono_start/stop(tcp_write_xmit,tcp_cwnd_validate,tcp_send_syn_data)
u8:2 chrono_type read_write tcp_chrono_start/stop(tcp_write_xmit,tcp_cwnd_validate,tcp_send_syn_data)
+u8 tcp_nospace read_mostly read_mostly tcp_check_space(tx);tcp_check_space(rx)
u8:1 rate_app_limited read_write tcp_rate_gen
u8:1 fastopen_connect
u8:1 fastopen_no_cookie
diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index 6a8c77719322f9caee305d954a107892c76d4ef7..e4d1720f49f08f3c12f5ba66fe8fe3379d71b1e0 100644
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -269,6 +269,9 @@ struct tcp_sock {
*/
u32 snd_sml; /* Last byte of the most recently transmitted small packet */
u8 chrono_type; /* current chronograph type */
+ u8 tcp_nospace; /* mirrors SOCK_NOSPACE, must be set whenever
+ * SOCK_NOSPACE is set.
+ */
u32 chrono_start; /* Start time in jiffies of a TCP chrono */
u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */
u32 write_seq; /* Tail(+1) of data held in tcp send buffer */diff --git a/include/net/tcp.h b/include/net/tcp.h
index 96a1a23e47fac9b2f8717a4e129f468d40c6cfb5..08c04f7b32960e66ae5885dc93710569abda62e5 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -783,14 +783,42 @@ void tcp_done_with_error(struct sock *sk, int err);
void tcp_reset(struct sock *sk, struct sk_buff *skb);
void tcp_fin(struct sock *sk);
void __tcp_check_space(struct sock *sk);
+
+/* Mirror of SOCK_NOSPACE in tcp_sock, maintained by sk_set_nospace()
+ * and sk_clear_nospace().
+ *
+ * MPTCP subflows share the parent socket, and thus its SOCK_NOSPACE bit.
+ * Keep their mirror always set (see subflow_ulp_init()) so that they
+ * always reach __tcp_check_space() and behave as before.
+ */
+static inline void tcp_set_nospace(struct sock *sk)
+{
+ if (sk_is_tcp(sk)) {
+ /* pairs with smp_mb__before_atomic() in tcp_clear_nospace() */
+ smp_mb__after_atomic();
+ /* pairs with smp_mb() in tcp_check_space() */
+ smp_store_mb(tcp_sk(sk)->tcp_nospace, 1);
+ }
+}
+
+static inline void tcp_clear_nospace(struct sock *sk)
+{
+ if (sk_is_tcp(sk) && !sk_is_mptcp(sk)) {
+ WRITE_ONCE(tcp_sk(sk)->tcp_nospace, 0);
+ /* pairs with smp_mb__after_atomic() in tcp_set_nospace() */
+ smp_mb__before_atomic();
+ }
+}
+
static inline void tcp_check_space(struct sock *sk)
{
/* pairs with tcp_poll() */
smp_mb();
- if (sk->sk_socket && test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
+ if (unlikely(READ_ONCE(tcp_sk(sk)->tcp_nospace)))
__tcp_check_space(sk);
}
+
void tcp_sack_compress_send_ack(struct sock *sk);
static inline void tcp_cleanup_skb(struct sk_buff *skb)diff --git a/net/core/sock.c b/net/core/sock.c
index 11a22aec7e414152aab115e8d11e30067ab3775f..946f614f665b3608d1f310da908d82de622ec5c0 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2969,8 +2969,15 @@ void sk_set_nospace(struct sock *sk)
{
struct socket *sock = sk->sk_socket;
- if (sock)
- set_bit(SOCK_NOSPACE, &sock->flags);
+ if (!sock)
+ return;
+ /* Set SOCK_NOSPACE before tp->tcp_nospace (paired with
+ * sk_clear_nospace() clearing tp->tcp_nospace before SOCK_NOSPACE)
+ * so a concurrent clear cannot leave SOCK_NOSPACE set with
+ * tp->tcp_nospace cleared.
+ */
+ set_bit(SOCK_NOSPACE, &sock->flags);
+ tcp_set_nospace(sk);
}
EXPORT_SYMBOL(sk_set_nospace);
@@ -2985,8 +2992,10 @@ void sk_clear_nospace(struct sock *sk)
{
struct socket *sock = sk->sk_socket;
- if (sock)
- clear_bit(SOCK_NOSPACE, &sock->flags);
+ if (!sock)
+ return;
+ tcp_clear_nospace(sk);
+ clear_bit(SOCK_NOSPACE, &sock->flags);
}
EXPORT_SYMBOL(sk_clear_nospace);
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 1cde000cfab4704e6756872f6ddec16851ccc55d..f44cbd3e76178a5ec5b40e420b32836477c29266 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -5242,6 +5242,7 @@ static void __init tcp_struct_check(void)
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, first_tx_mstamp);
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_mstamp);
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
+ CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_nospace);
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq);
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 892ff256e235272a8483a11949cc98b319dd7cc9..cdea30d07fe2c2a38c3d38da27ad3274221d5958 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -6098,8 +6098,14 @@ static void tcp_new_space(struct sock *sk)
*/
void __tcp_check_space(struct sock *sk)
{
+ struct socket *sock = sk->sk_socket;
+
+ /* tp->tcp_nospace is only a hint, SOCK_NOSPACE is authoritative. */
+ if (!sock || !test_bit(SOCK_NOSPACE, &sock->flags))
+ return;
+
tcp_new_space(sk);
- if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
+ if (!test_bit(SOCK_NOSPACE, &sock->flags))
tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
}
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index f0a6725d2c3762def75c879e769e9da37f92215a..e297c88be503b42979886baaf43acfb6b7e78055 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -2001,6 +2001,11 @@ static int subflow_ulp_init(struct sock *sk)
pr_debug("subflow=%p, family=%d\n", ctx, sk->sk_family);
tp->is_mptcp = 1;
+ /* Subflows share the MPTCP socket, and thus its SOCK_NOSPACE bit,
+ * which tcp_check_space() can not mirror. Pin the mirror so that
+ * __tcp_check_space() always tests the shared bit.
+ */
+ tp->tcp_nospace = 1;
ctx->icsk_af_ops = icsk->icsk_af_ops;
icsk->icsk_af_ops = subflow_default_af_ops(sk);
ctx->tcp_state_change = sk->sk_state_change;--
2.56.0.rc1.315.gc6ed9934b7-goog