[PATCH 10/10] drivers/infiniband/hw/virtio: Add completion queue notification support
From: Xiong Weimin <hidden>
Date: 2025-12-18 09:11:46
Also in:
kvm, virtualization
Subsystem:
the rest · Maintainer:
Linus Torvalds
From: xiongweimin <redacted>
This commit implements CQ (Completion Queue) notification functionality
for virtio RDMA devices:
1. Notification types:
- Solicited completion notifications (IB_CQ_SOLICITED)
- Next completion notifications (IB_CQ_NEXT_COMP)
- Error handling for unsupported flags
2. Backend communication:
- VIRTIO_RDMA_CMD_REQ_NOTIFY_CQ command implementation
- Command/response buffer management
- Error handling for virtqueue operations
3. Resource management:
- Dynamic memory allocation for command/response structs
- Guaranteed cleanup on error paths
- Rate-limited error logging
4. Feature limitations:
- REPORT_MISSED_EVENTS currently returns -EOPNOTSUPP
(to be implemented in future work)
Signed-off-by: Xiong Weimin <redacted>
---
.../drivers/infiniband/hw/virtio/vrdma_abi.h | 6 +
.../infiniband/hw/virtio/vrdma_dev_api.h | 9 ++
.../drivers/infiniband/hw/virtio/vrdma_ib.c | 121 +++++++++++++++++-
3 files changed, 135 insertions(+), 1 deletion(-)
diff --git a/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_abi.h b/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_abi.h
index 0a9404057..ff4b2505f 100644
--- a/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_abi.h
+++ b/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_abi.h@@ -9,6 +9,12 @@ #define VRDMA_ABI_VERSION 1 +enum { + VIRTIO_RDMA_NOTIFY_NOT = (0), + VIRTIO_RDMA_NOTIFY_SOLICITED = (1 << 0), + VIRTIO_RDMA_NOTIFY_NEXT_COMPLETION = (1 << 1) +}; + /** * struct vrdma_cqe - Virtio-RDMA Completion Queue Entry (CQE) *
diff --git a/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_dev_api.h b/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_dev_api.h
index 86b5ecade..d9a65531e 100644
--- a/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_dev_api.h
+++ b/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_dev_api.h@@ -243,6 +243,15 @@ struct vrdma_rsp_query_qp { struct vrdma_qp_attr attr; }; +struct vrdma_cmd_req_notify { + __u32 cqn; + __u32 flags; +}; + +struct vrdma_rsp_req_notify { + __u32 missed; +}; + struct vrdma_cmd_reg_user_mr { __u32 pdn; __u32 access_flags;
diff --git a/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_ib.c b/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_ib.c
index b1429e072..6f97c6bdc 100644
--- a/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_ib.c
+++ b/linux-6.16.8/drivers/infiniband/hw/virtio/vrdma_ib.c@@ -2849,6 +2849,116 @@ static struct ib_mr *vrdma_reg_user_mr(struct ib_pd *pd, u64 start, return ERR_PTR(rc); } +/** + * vrdma_req_notify_cq - Request notification for CQ events + * @ibcq: Completion queue + * @flags: Notification flags (e.g., solicited only, report missed) + * + * Requests that the next completion (or next solicited completion) trigger + * an interrupt/event. Also supports checking if events were missed. + * + * Context: Process context (may sleep). Called from user or kernel path. + * Return: + * * 0 on success + * * -EOPNOTSUPP if unsupported flag (e.g., REPORT_MISSED_EVENTS) + * * -ENOMEM if command allocation fails + * * -EIO if communication with backend fails + */ +static int vrdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + struct vrdma_cq *vcq = to_vcq(ibcq); + struct vrdma_dev *vdev = to_vdev(ibcq->device); + struct vrdma_cmd_req_notify *cmd; + struct vrdma_rsp_req_notify *rsp; + struct scatterlist in, out; + int rc = 0; + + /* Handle solicited-only or any-completion notification */ + if (flags & IB_CQ_SOLICITED_MASK) { + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + rsp = kzalloc(sizeof(*rsp), GFP_KERNEL); + if (!rsp) { + kfree(cmd); + return -ENOMEM; + } + + cmd->cqn = cpu_to_le32(vcq->cq_handle); + + if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) + cmd->flags = VIRTIO_RDMA_NOTIFY_SOLICITED; + else + cmd->flags = VIRTIO_RDMA_NOTIFY_NEXT_COMPLETION; + + sg_init_one(&in, cmd, sizeof(*cmd)); + sg_init_one(&out, rsp, sizeof(*rsp)); + + rc = vrdma_exec_verbs_cmd(vdev, VIRTIO_RDMA_CMD_REQ_NOTIFY_CQ, + &in, &out); + + if (rc) { + dev_err(&vdev->vdev->dev, + "VIRTIO_RDMA_CMD_REQ_NOTIFY_CQ failed: cqn=%u, rc=%d\n", + vcq->cq_handle, rc); + rc = -EIO; + } + + kfree(rsp); + kfree(cmd); + + if (rc) + return rc; + } + + /* + * Check for missed events: this requires querying backend state + * Currently not supported in most virtio-rdma implementations. + */ + if (flags & IB_CQ_REPORT_MISSED_EVENTS) { + /* + * Ideally we'd query the host whether an event has occurred + * since last notify, but this is often unimplemented. + */ + return -EOPNOTSUPP; + } + + return 0; +} + +static ssize_t hca_type_show(struct device *device, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "VIRTIO-RDMA-%s\n", VIRTIO_RDMA_DRIVER_VER); +} +static DEVICE_ATTR_RO(hca_type); + +static ssize_t hw_rev_show(struct device *device, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", VIRTIO_RDMA_HW_REV); +} +static DEVICE_ATTR_RO(hw_rev); + +static ssize_t board_id_show(struct device *device, + struct device_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", VIRTIO_RDMA_BOARD_ID); +} +static DEVICE_ATTR_RO(board_id); + +static struct attribute *vrdma_class_attributes[] = { + &dev_attr_hw_rev.attr, + &dev_attr_hca_type.attr, + &dev_attr_board_id.attr, + NULL, +}; + +static const struct attribute_group vrdma_attr_group = { + .attrs = vrdma_class_attributes, +}; + static const struct ib_device_ops vrdma_dev_ops = { .owner = THIS_MODULE, .uverbs_abi_ver = VIRTIO_RDMA_ABI_VERSION,
@@ -2885,7 +2995,16 @@ static const struct ib_device_ops vrdma_dev_ops = { .post_send = vrdma_post_send, .query_pkey = vrdma_query_pkey, .query_qp = vrdma_query_qp, - .reg_user_mr = vrdma_reg_user_mr, + .reg_user_mr = vrdma_reg_user_mr, + .req_notify_cq = vrdma_req_notify_cq, + + .device_group = &vrdma_attr_group, + + INIT_RDMA_OBJ_SIZE(ib_ah, vrdma_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, vrdma_cq, ibcq), + INIT_RDMA_OBJ_SIZE(ib_pd, vrdma_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_qp, vrdma_qp, ibqp), + INIT_RDMA_OBJ_SIZE(ib_ucontext, vrdma_ucontext, ibucontext), }; /**
--
2.43.0