diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 818efdeccef1..0458a9691cbc 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -175,6 +175,7 @@ struct nvmet_tcp_queue {
/* TLS state */
key_serial_t tls_pskid;
+ key_serial_t handshake_session_id;
struct delayed_work tls_handshake_tmo_work;
unsigned long poll_end;@@ -186,6 +187,8 @@ struct nvmet_tcp_queue {
struct sockaddr_storage sockaddr_peer;
struct work_struct release_work;
+ struct completion tls_complete;
+
int idx;
struct list_head queue_list;
@@ -214,6 +217,10 @@ static struct workqueue_struct *nvmet_tcp_wq;
static const struct nvmet_fabrics_ops nvmet_tcp_ops;
static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c);
static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd);
+#ifdef CONFIG_NVME_TARGET_TCP_TLS
+static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue,
+ enum handshake_key_update_type keyupdate);
+#endif
static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue,
struct nvmet_tcp_cmd *cmd)
@@ -832,6 +839,20 @@ static int nvmet_tcp_try_send_one(struct nvmet_tcp_queue *queue,
return 1;
}
+#ifdef CONFIG_NVME_TARGET_TCP_TLS
+static bool nvmet_tls_key_expired(struct nvmet_tcp_queue *queue, int ret)
+{
+ return ret == -EKEYEXPIRED &&
+ queue->state != NVMET_TCP_Q_DISCONNECTING &&
+ queue->state != NVMET_TCP_Q_TLS_HANDSHAKE;
+}
+#else
+static bool nvmet_tls_key_expired(struct nvmet_tcp_queue *queue, int ret)
+{
+ return false;
+}
+#endif
+
static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue,
int budget, int *sends)
{@@ -1106,6 +1127,103 @@ static inline bool nvmet_tcp_pdu_valid(u8 type)
return false;
}
+static void nvmet_tcp_release_queue(struct kref *kref)
+{
+ struct nvmet_tcp_queue *queue =
+ container_of(kref, struct nvmet_tcp_queue, kref);
+
+ WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING);
+ queue_work(nvmet_wq, &queue->release_work);
+}
+
+static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
+{
+ spin_lock_bh(&queue->state_lock);
+ if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
+ /* Socket closed during handshake */
+ tls_handshake_cancel(queue->sock->sk);
+ }
+ if (queue->state != NVMET_TCP_Q_DISCONNECTING) {
+ queue->state = NVMET_TCP_Q_DISCONNECTING;
+ kref_put(&queue->kref, nvmet_tcp_release_queue);
+ }
+ spin_unlock_bh(&queue->state_lock);
+}
+
+static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
+{
+ struct socket *sock = queue->sock;
+
+ if (!queue->state_change)
+ return;
+
+ write_lock_bh(&sock->sk->sk_callback_lock);
+ sock->sk->sk_data_ready = queue->data_ready;
+ sock->sk->sk_state_change = queue->state_change;
+ sock->sk->sk_write_space = queue->write_space;
+ sock->sk->sk_user_data = NULL;
+ write_unlock_bh(&sock->sk->sk_callback_lock);
+}
+
+#ifdef CONFIG_NVME_TARGET_TCP_TLS
+static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w)
+{
+ struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w),
+ struct nvmet_tcp_queue, tls_handshake_tmo_work);
+
+ pr_warn("queue %d: TLS handshake timeout\n", queue->idx);
+ /*
+ * If tls_handshake_cancel() fails we've lost the race with
+ * nvmet_tcp_tls_handshake_done() */
+ if (!tls_handshake_cancel(queue->sock->sk))
+ return;
+ spin_lock_bh(&queue->state_lock);
+ if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) {
+ spin_unlock_bh(&queue->state_lock);
+ return;
+ }
+ queue->state = NVMET_TCP_Q_FAILED;
+ spin_unlock_bh(&queue->state_lock);
+ nvmet_tcp_schedule_release_queue(queue);
+ kref_put(&queue->kref, nvmet_tcp_release_queue);
+}
+
+static int update_tls_keys(struct nvmet_tcp_queue *queue)
+{
+ int ret;
+
+ cancel_work(&queue->io_work);
+ queue->state = NVMET_TCP_Q_TLS_HANDSHAKE;
+
+ nvmet_tcp_restore_socket_callbacks(queue);
+
+ INIT_DELAYED_WORK(&queue->tls_handshake_tmo_work,
+ nvmet_tcp_tls_handshake_timeout);
+
+ ret = nvmet_tcp_tls_handshake(queue, HANDSHAKE_KEY_UPDATE_TYPE_RECEIVED);
+
+ if (ret < 0)
+ return ret;
+
+ ret = wait_for_completion_interruptible_timeout(&queue->tls_complete,
+ 10 * HZ);
+
+ if (ret <= 0) {
+ tls_handshake_cancel(queue->sock->sk);
+ return ret;
+ }
+
+ queue->state = NVMET_TCP_Q_LIVE;
+
+ return 0;
+}
+#else
+static int update_tls_keys(struct nvmet_tcp_queue *queue)
+{
+ return -EPFNOSUPPORT;
+}
+#endif
+
static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue,
struct msghdr *msg, char *cbuf)
{@@ -1131,6 +1249,9 @@ static int nvmet_tcp_tls_record_ok(struct nvmet_tcp_queue *queue,
ret = -EAGAIN;
}
break;
+ case TLS_RECORD_TYPE_HANDSHAKE:
+ ret = -EAGAIN;
+ break;
default:
/* discard this record type */
pr_err("queue %d: TLS record %d unhandled\n",@@ -1340,6 +1461,8 @@ static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,
for (i = 0; i < budget; i++) {
ret = nvmet_tcp_try_recv_one(queue);
if (unlikely(ret < 0)) {
+ if (nvmet_tls_key_expired(queue, ret))
+ goto done;
nvmet_tcp_socket_error(queue, ret);
goto done;
} else if (ret == 0) {@@ -1351,29 +1474,6 @@ static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue,
return ret;
}
-static void nvmet_tcp_release_queue(struct kref *kref)
-{
- struct nvmet_tcp_queue *queue =
- container_of(kref, struct nvmet_tcp_queue, kref);
-
- WARN_ON(queue->state != NVMET_TCP_Q_DISCONNECTING);
- queue_work(nvmet_wq, &queue->release_work);
-}
-
-static void nvmet_tcp_schedule_release_queue(struct nvmet_tcp_queue *queue)
-{
- spin_lock_bh(&queue->state_lock);
- if (queue->state == NVMET_TCP_Q_TLS_HANDSHAKE) {
- /* Socket closed during handshake */
- tls_handshake_cancel(queue->sock->sk);
- }
- if (queue->state != NVMET_TCP_Q_DISCONNECTING) {
- queue->state = NVMET_TCP_Q_DISCONNECTING;
- kref_put(&queue->kref, nvmet_tcp_release_queue);
- }
- spin_unlock_bh(&queue->state_lock);
-}
-
static inline void nvmet_tcp_arm_queue_deadline(struct nvmet_tcp_queue *queue)
{
queue->poll_end = jiffies + usecs_to_jiffies(idle_poll_period_usecs);@@ -1404,8 +1504,12 @@ static void nvmet_tcp_io_work(struct work_struct *w)
ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops);
if (ret > 0)
pending = true;
- else if (ret < 0)
+ else if (ret < 0) {
+ if (ret == -EKEYEXPIRED)
+ break;
+
return;
+ }
ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops);
if (ret > 0)@@ -1415,6 +1519,11 @@ static void nvmet_tcp_io_work(struct work_struct *w)
} while (pending && ops < NVMET_TCP_IO_WORK_BUDGET);
+ if (ret == -EKEYEXPIRED) {
+ update_tls_keys(queue);
+ pending = true;
+ }
+
/*
* Requeue the worker if idle deadline period is in progress or any
* ops activity was recorded during the do-while loop above.@@ -1517,21 +1626,6 @@ static void nvmet_tcp_free_cmds(struct nvmet_tcp_queue *queue)
kfree(cmds);
}
-static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue)
-{
- struct socket *sock = queue->sock;
-
- if (!queue->state_change)
- return;
-
- write_lock_bh(&sock->sk->sk_callback_lock);
- sock->sk->sk_data_ready = queue->data_ready;
- sock->sk->sk_state_change = queue->state_change;
- sock->sk->sk_write_space = queue->write_space;
- sock->sk->sk_user_data = NULL;
- write_unlock_bh(&sock->sk->sk_callback_lock);
-}
-
static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue)
{
struct nvmet_tcp_cmd *cmd = queue->cmds;@@ -1794,6 +1888,7 @@ static void nvmet_tcp_tls_handshake_done(void *data, int status,
}
if (!status) {
queue->tls_pskid = peerid;
+ queue->handshake_session_id = handshake_session_id;
queue->state = NVMET_TCP_Q_CONNECTING;
} else
queue->state = NVMET_TCP_Q_FAILED;@@ -1809,28 +1904,7 @@ static void nvmet_tcp_tls_handshake_done(void *data, int status,
else
nvmet_tcp_set_queue_sock(queue);
kref_put(&queue->kref, nvmet_tcp_release_queue);
-}
-
-static void nvmet_tcp_tls_handshake_timeout(struct work_struct *w)
-{
- struct nvmet_tcp_queue *queue = container_of(to_delayed_work(w),
- struct nvmet_tcp_queue, tls_handshake_tmo_work);
-
- pr_warn("queue %d: TLS handshake timeout\n", queue->idx);
- /*
- * If tls_handshake_cancel() fails we've lost the race with
- * nvmet_tcp_tls_handshake_done() */
- if (!tls_handshake_cancel(queue->sock->sk))
- return;
- spin_lock_bh(&queue->state_lock);
- if (WARN_ON(queue->state != NVMET_TCP_Q_TLS_HANDSHAKE)) {
- spin_unlock_bh(&queue->state_lock);
- return;
- }
- queue->state = NVMET_TCP_Q_FAILED;
- spin_unlock_bh(&queue->state_lock);
- nvmet_tcp_schedule_release_queue(queue);
- kref_put(&queue->kref, nvmet_tcp_release_queue);
+ complete(&queue->tls_complete);
}
static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue,@@ -1852,11 +1926,15 @@ static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue,
args.ta_data = queue;
args.ta_keyring = key_serial(queue->port->nport->keyring);
args.ta_timeout_ms = tls_handshake_timeout * 1000;
+ args.ta_handshake_session_id = queue->handshake_session_id;
+
+ init_completion(&queue->tls_complete);
if (keyupdate == HANDSHAKE_KEY_UPDATE_TYPE_UNSPEC)
ret = tls_server_hello_psk(&args, GFP_KERNEL);
else
ret = tls_server_keyupdate_psk(&args, GFP_KERNEL, keyupdate);
+
if (ret) {
kref_put(&queue->kref, nvmet_tcp_release_queue);
pr_err("failed to start TLS, err=%d\n", ret);--
2.51.1