-----Original Message-----
From: Jiri Pirko <jiri@resnulli.us>
Sent: Tuesday, March 18, 2025 5:47 AM
To: <redacted>
Cc: davem@davemloft.net; Dumazet, Eric <redacted>;
kuba@kernel.org; pabeni@redhat.com; saeedm@nvidia.com; leon@kernel.org;
tariqt@nvidia.com; andrew+netdev@lunn.ch; dakr@kernel.org;
rafael@kernel.org; gregkh@linuxfoundation.org; Kitszel, Przemyslaw
[off-list ref]; Nguyen, Anthony L
[off-list ref]; cratiu@nvidia.com; Keller, Jacob E
[off-list ref]; Knitter, Konrad [off-list ref];
cjubran@nvidia.com
Subject: [PATCH net-next RFC 2/3] net/mlx5: Introduce shared devlink instance for
PFs on same chip
From: Jiri Pirko <redacted>
Multiple PFS may reside on the same physical chip, running a single
firmware. Some of the resources and configurations may be shared among
these PFs. Currently, there is not good object to pin the configuration
knobs on.
Introduce a shared devlink, instantiated upon probe of the first PF,
removed during remove of the last PF. Back this shared devlink instance
by faux device, as there is no PCI device related to it.
Make the PF devlink instances nested in this shared devlink instance.
Example:
$ devlink dev
pci/0000:08:00.0:
nested_devlink:
auxiliary/mlx5_core.eth.0
faux/mlx5_core_83013c12b77faa1a30000c82a1045c91:
nested_devlink:
pci/0000:08:00.0
pci/0000:08:00.1
auxiliary/mlx5_core.eth.0
pci/0000:08:00.1:
nested_devlink:
auxiliary/mlx5_core.eth.1
auxiliary/mlx5_core.eth.1
Signed-off-by: Jiri Pirko <redacted>
---
.../net/ethernet/mellanox/mlx5/core/Makefile | 4 +-
.../net/ethernet/mellanox/mlx5/core/main.c | 18 +++
.../ethernet/mellanox/mlx5/core/sh_devlink.c | 150 ++++++++++++++++++
.../ethernet/mellanox/mlx5/core/sh_devlink.h | 10 ++
include/linux/mlx5/driver.h | 5 +
5 files changed, 185 insertions(+), 2 deletions(-)
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 568bbe5f83f5..510850b6e6e2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -16,8 +16,8 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o
pagealloc.o \
transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \
fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o
events.o wq.o lib/gid.o \
lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o
diag/fs_tracepoint.o \
- diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o
diag/reporter_vnic.o \
- fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o
+ diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o
diag/rsc_dump.o \
+ diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o
fs_pool.o
#
# Netdev basic
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c
b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 710633d5fdbe..e1217a8bf4db 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -74,6 +74,7 @@
#include "mlx5_irq.h"
#include "hwmon.h"
#include "lag/lag.h"
+#include "sh_devlink.h"
MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectXseries) core driver");
@@ -1554,10 +1555,17 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
int err;
devl_lock(devlink);
+ if (dev->shd) {
+ err = devl_nested_devlink_set(priv_to_devlink(dev->shd),
+ devlink);
+ if (err)
+ goto unlock;
+ }
devl_register(devlink);
err = mlx5_init_one_devl_locked(dev);
if (err)
devl_unregister(devlink);
+unlock:
devl_unlock(devlink);
return err;
}@@ -1998,6 +2006,13 @@ static int probe_one(struct pci_dev *pdev, const struct
pci_device_id *id)
goto pci_init_err;
}
+ err = mlx5_shd_init(dev);
+ if (err) {
+ mlx5_core_err(dev, "mlx5_shd_init failed with error code %d\n",
+ err);
+ goto shd_init_err;
+ }
+
err = mlx5_init_one(dev);
if (err) {
mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n",@@ -2009,6 +2024,8 @@ static int probe_one(struct pci_dev *pdev, const struct
pci_device_id *id)
return 0;
err_init_one:
+ mlx5_shd_uninit(dev);
+shd_init_err:
mlx5_pci_close(dev);
pci_init_err:
mlx5_mdev_uninit(dev);
@@ -2030,6 +2047,7 @@ static void remove_one(struct pci_dev *pdev)
mlx5_drain_health_wq(dev);
mlx5_sriov_disable(pdev, false);
mlx5_uninit_one(dev);
+ mlx5_shd_uninit(dev);
mlx5_pci_close(dev);
mlx5_mdev_uninit(dev);
mlx5_adev_idx_free(dev->priv.adev_idx);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
new file mode 100644
index 000000000000..671a3442525b
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c
@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include <linux/device/faux.h>
+#include <linux/mlx5/driver.h>
+#include <linux/mlx5/vport.h>
+
+#include "sh_devlink.h"
+
+static LIST_HEAD(shd_list);
+static DEFINE_MUTEX(shd_mutex); /* Protects shd_list and shd->list */
+
+/* This structure represents a shared devlink instance,
+ * there is one created for PF group of the same chip.
+ */
+struct mlx5_shd {
+ /* Node in shd list */
+ struct list_head list;
+ /* Serial number of the chip */
+ const char *sn;
+ /* List of per-PF dev instances. */
+ struct list_head dev_list;
+ /* Related faux device */
+ struct faux_device *faux_dev;
+};
+