5. tcp_write_xmit() leaves the corked record unsent with packets_out == 0, so
     tcp_check_probe_timer() arms the persist (probe-0) timer even though the
     window is open; each expiry pushes ~one record -> the observed rate. At a
     1448-byte MSS each 16 KB record already exceeds MSS and is sent, so the
     cork never engages.

Candidate fix (illustrative sketch -- NOT a submittable patch, no S-o-b; the
right shape is your call given the coalescing intent). A full TLS record is a
natural transmit boundary, so arguably MSG_MORE should not be honoured for it,
only for a trailing partial record. In tls_sw_sendmsg_locked():

    if (full_record || eor) {
+           unsigned int send_flags = msg->msg_flags;
+
+           if (full_record)
+                   send_flags &= ~MSG_MORE;
            ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
-                                     record_type, &copied, msg->msg_flags);
+                                     record_type, &copied, send_flags);

Caveats I'm aware of: (a) there are two bpf_exec_tx_verdict() call sites in the
function passing msg->msg_flags -- the splice path reaches the one at `copied:`
label (via `goto copied`), so this one-site change covers the repro, but a
complete fix would clear MSG_MORE at both; (b) this stops coalescing each
record's trailing partial into the next TSO segment, partly undoing the 2023
optimisation, so a narrower fix that only flushes when about to block in
sk_stream_wait_memory() may be preferable. Happy to spin whichever you prefer
and run it through the tls selftests.

Thanks,
Mike Fara

--- ktls_sendfile_stall.c ---

// ktls_sendfile_stall.c
//
// Minimal, dependency-free reproducer for a software-kTLS (TLS_SW) TX stall:
// sendfile()/splice() over a kTLS socket collapses to the TCP persist-timer
// cadence when the path MSS is large (loopback's 65483, or jumbo frames),
// because splice sets MSG_MORE on every chunk but the last and tls_sw forwards
// it to TCP even for *completed* records, so TCP corks the sub-MSS record while
// tls_sw blocks in sk_stream_wait_memory().
//
// No handshake / no OpenSSL: TLS_TX is programmed with a fixed key (the receiver
// discards ciphertext; we only measure the TX path), exactly like the in-tree
// tls selftest. Clamping the sender MSS to a realistic value makes the stall
// vanish, isolating the large-MSS amplifier.
//
//   cc -O2 -Wall -o ktls_sendfile_stall ktls_sendfile_stall.c
//   ./ktls_sendfile_stall          # default loopback MSS (65483) -> stalls
//   ./ktls_sendfile_stall 1448     # clamp sender MSS via TCP_MAXSEG -> normal
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <signal.h>
#include <time.h>
#include <sys/socket.h>
#include <sys/sendfile.h>
#include <sys/wait.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <linux/tls.h>

#ifndef SOL_TLS
#define SOL_TLS 282
#endif
#ifndef TCP_ULP
#define TCP_ULP 31
#endif

#define PORT     18999
#define FILESZ   (4 * 1024 * 1024)            /* 4 MiB backing file, looped     */
#define TOTAL    (2ULL * 1024 * 1024 * 1024)  /* try to send 2 GiB              */
#define STALL_S  20.0                         /* give up after this many seconds */

static void die(const char *what) { perror(what); exit(1); }

static double now(void)
{
	struct timespec t;
	clock_gettime(CLOCK_MONOTONIC, &t);
	return t.tv_sec + t.tv_nsec / 1e9;
}

static void set_ktls_tx(int fd)
{
	struct tls12_crypto_info_aes_gcm_128 ci;

	if (setsockopt(fd, IPPROTO_TCP, TCP_ULP, "tls", sizeof("tls")))
		die("setsockopt(TCP_ULP, tls)");
	memset(&ci, 0, sizeof(ci));
	ci.info.version     = TLS_1_3_VERSION;          /* matches the 16406 ss line */
	ci.info.cipher_type = TLS_CIPHER_AES_GCM_128;
	memset(ci.iv,      1, sizeof(ci.iv));
	memset(ci.key,     2, sizeof(ci.key));
	memset(ci.salt,    3, sizeof(ci.salt));
	memset(ci.rec_seq, 0, sizeof(ci.rec_seq));
	if (setsockopt(fd, SOL_TLS, TLS_TX, &ci, sizeof(ci)))
		die("setsockopt(SOL_TLS, TLS_TX)");
}

int main(int argc, char **argv)
{
	int mss = (argc > 1) ? atoi(argv[1]) : 0;   /* 0 == leave default; >0 clamps */
	char path[] = "/tmp/ktls_repro_dataXXXXXX";
	struct sockaddr_in a;
	char *buf;
	int ffd, lfd, s, one = 1;
	pid_t pid;
	double t0, el;
	unsigned long long sent = 0;
	off_t off = 0;

	/* A dead/RST'd receiver must surface as EPIPE from sendfile(), not a
	 * silent SIGPIPE kill that would make the reproducer look broken. */
	signal(SIGPIPE, SIG_IGN);

	/* backing file */
	ffd = mkstemp(path);
	if (ffd < 0)
		die("mkstemp");
	buf = malloc(FILESZ);
	if (!buf)
		die("malloc");
	memset(buf, 'x', FILESZ);
	if (write(ffd, buf, FILESZ) != FILESZ)
		die("write");
	free(buf);

	memset(&a, 0, sizeof(a));
	a.sin_family      = AF_INET;
	a.sin_port        = htons(PORT);
	a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);

	lfd = socket(AF_INET, SOCK_STREAM, 0);
	if (lfd < 0)
		die("socket");
	setsockopt(lfd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
	if (mss > 0)   /* announce a small MSS in the SYN-ACK; inherited by accept() */
		setsockopt(lfd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss));
	if (bind(lfd, (void *)&a, sizeof(a)))
		die("bind");
	if (listen(lfd, 1))
		die("listen");

	pid = fork();
	if (pid < 0)
		die("fork");
	if (pid == 0) {                 /* receiver: connect, drain ciphertext, discard */
		int c = socket(AF_INET, SOCK_STREAM, 0);
		char *r = malloc(1 << 20);
		ssize_t n;

		if (c < 0 || !r)
			_exit(1);
		while (connect(c, (void *)&a, sizeof(a)))
			usleep(1000);
		while ((n = read(c, r, 1 << 20)) > 0)
			;
		_exit(0);
	}

	s = accept(lfd, 0, 0);
	if (s < 0)
		die("accept");
	set_ktls_tx(s);

	t0 = now();
	while (sent < TOTAL) {
		ssize_t n;
		size_t want;

		if (off >= FILESZ)
			off = 0;
		want = (size_t)(FILESZ - off);
		if (want > TOTAL - sent)
			want = TOTAL - sent;
		n = sendfile(s, ffd, &off, want);
		if (n < 0) {
			if (errno == EINTR)
				continue;
			perror("sendfile");
			break;
		}
		if (n == 0)
			break;
		sent += n;
		if (now() - t0 > STALL_S) {
			printf("[gave up after %.0fs]\n", STALL_S);
			break;
		}
	}
	el = now() - t0;
	printf("MSS=%-9s  sent=%8.1f MiB in %6.2fs  =>  %.4f GiB/s\n",
	       mss ? argv[1] : "default", sent / 1048576.0, el,
	       sent / el / (1024.0 * 1024 * 1024));

	close(s);
	kill(pid, SIGKILL);
	waitpid(pid, NULL, 0);
	unlink(path);
	return 0;
}