diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 4797a4532b0d..c0966929f97b 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -172,6 +172,7 @@ struct nvme_tcp_queue {
bool tls_enabled;
u32 rcv_crc;
u32 snd_crc;
+ key_serial_t handshake_session_id;
__le32 exp_ddgst;
__le32 recv_ddgst;
struct completion tls_complete;@@ -858,7 +859,10 @@ static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,
static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
{
char *pdu = queue->pdu;
+ char cbuf[CMSG_LEN(sizeof(char))] = {};
struct msghdr msg = {
+ .msg_control = cbuf,
+ .msg_controllen = sizeof(cbuf),
.msg_flags = MSG_DONTWAIT,
};
struct kvec iov = {@@ -873,6 +877,12 @@ static int nvme_tcp_recvmsg_pdu(struct nvme_tcp_queue *queue)
if (ret <= 0)
return ret;
+ if (cbuf[CMSG_LEN(sizeof(char)) - 1] == TLS_RECORD_TYPE_HANDSHAKE &&
+ pdu[queue->pdu_offset] == TLS_HANDSHAKE_KEYUPDATE) {
+ dev_err(queue->ctrl->ctrl.device, "KeyUpdate message\n");
+ return 1;
+ }
+
queue->pdu_remaining -= ret;
queue->pdu_offset += ret;
if (queue->pdu_remaining)@@ -944,14 +954,14 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
struct request *rq =
nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
+ char cbuf[CMSG_LEN(sizeof(char))] = {};
+ struct msghdr msg;
+ int ret;
if (nvme_tcp_recv_state(queue) != NVME_TCP_RECV_DATA)
return 0;
while (queue->data_remaining) {
- struct msghdr msg;
- int ret;
-
if (!iov_iter_count(&req->iter)) {
req->curr_bio = req->curr_bio->bi_next;
@@ -975,11 +985,8 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
msg.msg_flags = MSG_DONTWAIT;
ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
- if (ret < 0) {
- dev_err(queue->ctrl->ctrl.device,
- "queue %d failed to receive request %#x data",
- nvme_tcp_queue_id(queue), rq->tag);
- return ret;
+ if (unlikely(ret < 0)) {
+ goto msg_control;
}
if (queue->data_digest)
nvme_tcp_ddgst_calc(req, &queue->rcv_crc, ret);@@ -1002,6 +1009,29 @@ static int nvme_tcp_recvmsg_data(struct nvme_tcp_queue *queue)
}
}
+ return 0;
+
+msg_control:
+ /*
+ * If MSG_CTRUNC is set, it's a control message,
+ * so let's read the control message.
+ */
+ if (msg.msg_flags & MSG_CTRUNC) {
+ memset(&msg, 0, sizeof(msg));
+ msg.msg_flags = MSG_DONTWAIT;
+ msg.msg_control = cbuf;
+ msg.msg_controllen = sizeof(cbuf);
+
+ ret = sock_recvmsg(queue->sock, &msg, msg.msg_flags);
+ }
+
+ if (ret < 0) {
+ dev_err(queue->ctrl->ctrl.device,
+ "queue %d failed to receive request %#x data, %d",
+ nvme_tcp_queue_id(queue), rq->tag, ret);
+ return ret;
+ }
+
return 0;
}
@@ -1384,15 +1414,37 @@ static int nvme_tcp_try_recvmsg(struct nvme_tcp_queue *queue)
}
} while (result >= 0);
- if (result < 0 && result != -EAGAIN) {
- dev_err(queue->ctrl->ctrl.device,
- "receive failed: %d\n", result);
- queue->rd_enabled = false;
- nvme_tcp_error_recovery(&queue->ctrl->ctrl);
- } else if (result == -EAGAIN)
- result = 0;
+ if (result < 0) {
+ if (result != -EKEYEXPIRED && result != -EAGAIN) {
+ dev_err(queue->ctrl->ctrl.device,
+ "receive failed: %d\n", result);
+ queue->rd_enabled = false;
+ nvme_tcp_error_recovery(&queue->ctrl->ctrl);
+ }
+ return result;
+ }
- return result < 0 ? result : (queue->nr_cqe = nr_cqe);
+ queue->nr_cqe = nr_cqe;
+ return nr_cqe;
+}
+
+static void update_tls_keys(struct nvme_tcp_queue *queue)
+{
+ int qid = nvme_tcp_queue_id(queue);
+ int ret;
+
+ dev_dbg(queue->ctrl->ctrl.device,
+ "updating key for queue %d\n", qid);
+
+ ret = nvme_tcp_start_tls(&(queue->ctrl->ctrl),
+ queue, queue->ctrl->ctrl.tls_pskid,
+ HANDSHAKE_KEY_UPDATE_TYPE_RECEIVED);
+
+ if (ret < 0) {
+ dev_err(queue->ctrl->ctrl.device,
+ "failed to update the keys %d\n", ret);
+ nvme_tcp_fail_request(queue->request);
+ }
}
static void nvme_tcp_io_work(struct work_struct *w)@@ -1417,8 +1469,11 @@ static void nvme_tcp_io_work(struct work_struct *w)
result = nvme_tcp_try_recvmsg(queue);
if (result > 0)
pending = true;
- else if (unlikely(result < 0))
- return;
+ else if (unlikely(result < 0)) {
+ if (result == -EKEYEXPIRED)
+ update_tls_keys(queue);
+ break;
+ }
/* did we get some space after spending time in recv? */
if (nvme_tcp_queue_has_pending(queue) &&@@ -1726,6 +1781,7 @@ static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid,
ctrl->ctrl.tls_pskid = key_serial(tls_key);
key_put(tls_key);
queue->tls_err = 0;
+ queue->handshake_session_id = handshake_session_id;
}
out_complete:
@@ -1755,6 +1811,7 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
keyring = key_serial(nctrl->opts->keyring);
args.ta_keyring = keyring;
args.ta_timeout_ms = tls_handshake_timeout * 1000;
+ args.ta_handshake_session_id = queue->handshake_session_id;
queue->tls_err = -EOPNOTSUPP;
init_completion(&queue->tls_complete);
if (keyupdate == HANDSHAKE_KEY_UPDATE_TYPE_UNSPEC)
--
2.51.1