[PATCH net-next v10 07/12] enic: add MBOX core send and receive for admin channel
From: Satish Kharat <satishkh@cisco.com>
Date: 2026-06-29 17:26:25
Also in:
lkml
Subsystem:
cisco vic ethernet nic driver, networking drivers, the rest · Maintainers:
Satish Kharat, Andrew Lunn, "David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, Linus Torvalds
Implement the mailbox protocol engine used for PF-VF communication over the admin channel. The send path (enic_mbox_send_msg) builds a message with a common header, DMA-maps it, posts a single WQ descriptor with the destination vnic ID encoded in the VLAN tag field, and polls the WQ CQ for completion. MBOX sends are gated by enic->mbox_send_disabled: enic_mbox_send_msg() returns early while it is set. The flag is cleared in enic_admin_channel_open() only once the admin WQ/RQ/CQ and interrupt are fully programmed, and set again at the start of enic_admin_channel_close(), so a send can never race a not-yet-ready or torn-down admin channel. The receive path (enic_mbox_recv_handler) is installed as the admin RQ callback and validates incoming message headers. PF/VF-specific dispatch will be added in subsequent commits. Signed-off-by: Satish Kharat <satishkh@cisco.com> --- drivers/net/ethernet/cisco/enic/Makefile | 2 +- drivers/net/ethernet/cisco/enic/enic.h | 6 + drivers/net/ethernet/cisco/enic/enic_admin.c | 35 +++++- drivers/net/ethernet/cisco/enic/enic_mbox.c | 170 +++++++++++++++++++++++++++ drivers/net/ethernet/cisco/enic/enic_mbox.h | 8 ++ 5 files changed, 218 insertions(+), 3 deletions(-)
diff --git a/drivers/net/ethernet/cisco/enic/Makefile b/drivers/net/ethernet/cisco/enic/Makefile
index 7ae72fefc99a..e38aaf34c148 100644
--- a/drivers/net/ethernet/cisco/enic/Makefile
+++ b/drivers/net/ethernet/cisco/enic/Makefile@@ -4,5 +4,5 @@ obj-$(CONFIG_ENIC) := enic.o enic-y := enic_main.o vnic_cq.o vnic_intr.o vnic_wq.o \ enic_res.o enic_dev.o enic_pp.o vnic_dev.o vnic_rq.o vnic_vic.o \ enic_ethtool.o enic_api.o enic_clsf.o enic_rq.o enic_wq.o \ - enic_admin.o + enic_admin.o enic_mbox.o
diff --git a/drivers/net/ethernet/cisco/enic/enic.h b/drivers/net/ethernet/cisco/enic/enic.h
index 401123e6df1d..b009d87da4bd 100644
--- a/drivers/net/ethernet/cisco/enic/enic.h
+++ b/drivers/net/ethernet/cisco/enic/enic.h@@ -297,6 +297,8 @@ struct enic { * left the resources freed. */ bool admin_chan_up; + /* set on send timeout; cleared on channel re-open */ + bool mbox_send_disabled; struct vnic_wq admin_wq; struct vnic_rq admin_rq; struct vnic_cq admin_cq[2];
@@ -309,6 +311,10 @@ struct enic { unsigned int admin_msg_count; /* current depth of admin_msg_list */ void (*admin_rq_handler)(struct enic *enic, void *buf, unsigned int len); + + /* MBOX protocol state — mbox_lock serializes admin WQ sends */ + struct mutex mbox_lock; + u64 mbox_msg_num; }; static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev)
diff --git a/drivers/net/ethernet/cisco/enic/enic_admin.c b/drivers/net/ethernet/cisco/enic/enic_admin.c
index 6062a18043ba..d695b16765a1 100644
--- a/drivers/net/ethernet/cisco/enic/enic_admin.c
+++ b/drivers/net/ethernet/cisco/enic/enic_admin.c@@ -19,6 +19,7 @@ #include "cq_enet_desc.h" #include "wq_enet_desc.h" #include "rq_enet_desc.h" +#include "enic_mbox.h" /* Clean up any admin WQ buffers still held by hardware at close time. * Normally buffers are freed inline after send completion, but a timed-out
@@ -213,7 +214,26 @@ unsigned int enic_admin_rq_cq_service(struct enic *enic) goto next_desc; } - enic_admin_msg_enqueue(enic, buf->os_buf, bytes_written); + if (enic->admin_rq_handler) { + u16 sender_vlan; + + /* Firmware sets the CQ VLAN field to identify the + * sender: 0 = PF, 1-based = VF index. Overwrite + * the untrusted src_vnic_id in the MBOX header with + * the hardware-verified value. + */ + sender_vlan = le16_to_cpu(rq_desc->vlan); + if (bytes_written >= sizeof(struct enic_mbox_hdr)) { + struct enic_mbox_hdr *hdr = buf->os_buf; + + hdr->src_vnic_id = (sender_vlan == 0) ? + cpu_to_le16(ENIC_MBOX_DST_PF) : + cpu_to_le16(sender_vlan - 1); + } + + enic_admin_msg_enqueue(enic, buf->os_buf, + bytes_written); + } next_desc: enic_admin_rq_buf_clean(rq, rq->to_clean);
@@ -456,8 +476,9 @@ static void enic_admin_init_resources(struct enic *enic) VNIC_CQ_MSG_DISABLE, intr_offset, 0 /* cq_message_addr */); + /* coalescing_timer, coalescing_type, mask_on_assertion */ vnic_intr_init(&enic->admin_intr, - 0, 0, 1); /* coalescing_timer, coalescing_type, mask_on_assertion */ + 0, 0, 1); } static void enic_admin_msg_drain(struct enic *enic)
@@ -522,6 +543,14 @@ int enic_admin_channel_open(struct enic *enic) vnic_intr_unmask(&enic->admin_intr); + /* Only now that the admin WQ/RQ/CQ and interrupt are fully allocated, + * programmed and enabled is it safe to allow MBOX sends. Clearing this + * earlier opened a window where a concurrent sender (e.g. link-notify + * work scheduled by a post-reset link-up) could call enic_mbox_send_msg() + * against a not-yet-allocated admin_wq and crash. + */ + WRITE_ONCE(enic->mbox_send_disabled, false); + netdev_dbg(enic->netdev, "admin channel open: intr=%u wq_avail=%u rq_avail=%u cq0_color=%u cq1_color=%u\n", enic->admin_intr_index,
@@ -563,6 +592,8 @@ void enic_admin_channel_close(struct enic *enic) if (!enic->admin_chan_up) return; + WRITE_ONCE(enic->mbox_send_disabled, true); + netdev_dbg(enic->netdev, "admin channel close\n"); vnic_intr_mask(&enic->admin_intr);
diff --git a/drivers/net/ethernet/cisco/enic/enic_mbox.c b/drivers/net/ethernet/cisco/enic/enic_mbox.c
new file mode 100644
index 000000000000..3709704bee02
--- /dev/null
+++ b/drivers/net/ethernet/cisco/enic/enic_mbox.c@@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2025 Cisco Systems, Inc. All rights reserved. + +#include <linux/kernel.h> +#include <linux/netdevice.h> +#include <linux/dma-mapping.h> +#include <linux/delay.h> + +#include "vnic_dev.h" +#include "vnic_wq.h" +#include "vnic_cq.h" +#include "enic.h" +#include "enic_admin.h" +#include "enic_mbox.h" +#include "wq_enet_desc.h" + +#define ENIC_MBOX_POLL_TIMEOUT_US 5000000 +#define ENIC_MBOX_POLL_INTERVAL_US 100 + +static void enic_mbox_fill_hdr(struct enic *enic, struct enic_mbox_hdr *hdr, + u8 msg_type, u16 dst_vnic_id, u16 msg_len) +{ + memset(hdr, 0, sizeof(*hdr)); + hdr->dst_vnic_id = cpu_to_le16(dst_vnic_id); + hdr->msg_type = msg_type; + hdr->msg_len = cpu_to_le16(msg_len); + hdr->msg_num = cpu_to_le64(++enic->mbox_msg_num); +} + +int enic_mbox_send_msg(struct enic *enic, u8 msg_type, u16 dst_vnic_id, + void *payload, u16 payload_len) +{ + u16 total_len = sizeof(struct enic_mbox_hdr) + payload_len; + struct vnic_wq *wq = &enic->admin_wq; + struct wq_enet_desc *desc; + unsigned long timeout; + dma_addr_t dma_addr; + u16 vlan_tag; + void *buf; + int err; + + /* Serialize MBOX sends. The admin channel is a low-frequency + * control path; holding the mutex across the poll is acceptable. + */ + mutex_lock(&enic->mbox_lock); + + if (!enic->has_admin_channel || READ_ONCE(enic->mbox_send_disabled)) { + err = -ENODEV; + goto unlock; + } + + if (vnic_wq_desc_avail(wq) == 0) { + err = -ENOSPC; + goto unlock; + } + + buf = kmalloc(total_len, GFP_KERNEL); + if (!buf) { + err = -ENOMEM; + goto unlock; + } + + enic_mbox_fill_hdr(enic, buf, msg_type, dst_vnic_id, total_len); + if (payload_len) { + void *dst = buf + sizeof(struct enic_mbox_hdr); + + memcpy(dst, payload, payload_len); + } + + dma_addr = dma_map_single(&enic->pdev->dev, buf, total_len, + DMA_TO_DEVICE); + if (dma_mapping_error(&enic->pdev->dev, dma_addr)) { + kfree(buf); + err = -ENOMEM; + goto unlock; + } + + /* Firmware uses vlan field for routing: 0 = PF, 1-based = VF index */ + if (dst_vnic_id == ENIC_MBOX_DST_PF) + vlan_tag = 0; + else + vlan_tag = dst_vnic_id + 1; + + desc = vnic_wq_next_desc(wq); + wq_enet_desc_enc(desc, (u64)dma_addr | VNIC_PADDR_TARGET, + total_len, + 0, 0, 0, /* mss, hdr_len, offload_mode */ + 1, 1, /* eop, cq_entry */ + 0, /* fcoe_encap */ + 1, vlan_tag, /* vlan_tag_insert, vlan_tag */ + 0); /* loopback */ + vnic_wq_post(wq, buf, dma_addr, total_len, + 1, 1, /* sop, eop */ + 1, 1, /* desc_skip_cnt, cq_entry */ + 0, 0); /* compressed_send, wrid */ + vnic_wq_doorbell(wq); + + timeout = jiffies + usecs_to_jiffies(ENIC_MBOX_POLL_TIMEOUT_US); + err = -ETIMEDOUT; + while (time_before(jiffies, timeout)) { + if (enic_admin_wq_cq_service(enic)) { + err = 0; + break; + } + usleep_range(ENIC_MBOX_POLL_INTERVAL_US, + ENIC_MBOX_POLL_INTERVAL_US + 50); + } + /* Final check in case completion arrived during the last sleep */ + if (err && enic_admin_wq_cq_service(enic)) + err = 0; + + if (!err) { + wq->to_clean = wq->to_clean->next; + wq->ring.desc_avail++; + dma_unmap_single(&enic->pdev->dev, dma_addr, total_len, + DMA_TO_DEVICE); + kfree(buf); + } else { + netdev_err(enic->netdev, + "MBOX send timed out (type %u dst %u), disabling channel\n", + msg_type, dst_vnic_id); + /* + * The WQ descriptor is still live in hardware. Do not unmap + * or free the buffer: the device may still DMA from dma_addr. + * Mark the channel unusable so no further sends are attempted. + */ + WRITE_ONCE(enic->mbox_send_disabled, true); + } + + netdev_dbg(enic->netdev, + "MBOX send msg_type %u dst %u vlan %u err %d\n", + msg_type, dst_vnic_id, vlan_tag, err); +unlock: + mutex_unlock(&enic->mbox_lock); + return err; +} + +static void enic_mbox_recv_handler(struct enic *enic, void *buf, + unsigned int len) +{ + struct enic_mbox_hdr *hdr = buf; + + if (len < sizeof(*hdr)) { + if (net_ratelimit()) + netdev_warn(enic->netdev, + "MBOX: truncated message (len %u < %zu)\n", + len, sizeof(*hdr)); + return; + } + + if (hdr->msg_type >= ENIC_MBOX_MAX) { + if (net_ratelimit()) + netdev_warn(enic->netdev, + "MBOX: unknown msg type %u\n", + hdr->msg_type); + return; + } + + netdev_dbg(enic->netdev, + "MBOX recv: type %u from vnic %u len %u\n", + hdr->msg_type, le16_to_cpu(hdr->src_vnic_id), + le16_to_cpu(hdr->msg_len)); +} + +void enic_mbox_init(struct enic *enic) +{ + enic->mbox_msg_num = 0; + mutex_init(&enic->mbox_lock); + enic->admin_rq_handler = enic_mbox_recv_handler; +}
diff --git a/drivers/net/ethernet/cisco/enic/enic_mbox.h b/drivers/net/ethernet/cisco/enic/enic_mbox.h
index a52f1d25cb21..73fd7f783ee2 100644
--- a/drivers/net/ethernet/cisco/enic/enic_mbox.h
+++ b/drivers/net/ethernet/cisco/enic/enic_mbox.h@@ -80,4 +80,12 @@ struct enic_mbox_pf_link_state_ack_msg { struct enic_mbox_generic_reply ack; }; +#define ENIC_MBOX_DST_PF 0xFFFF + +struct enic; + +void enic_mbox_init(struct enic *enic); +int enic_mbox_send_msg(struct enic *enic, u8 msg_type, u16 dst_vnic_id, + void *payload, u16 payload_len); + #endif /* _ENIC_MBOX_H_ */
--
2.43.0