Re: [PATCH v13 5/5] RDMA/mlx5: get tph for p2p access when registering dma-buf mr

Peer-to-peer DMA between a mlx5 NIC and a foreign PCIe endpoint
(typically a GPU or a vfio-pci passthrough device) traverses the host
PCIe fabric. The endpoint exporting the dma-buf knows which PCIe TLP
Processing Hint (TPH) Steering Tag yields the best placement for the
traffic it will sink: per-endpoint hint selection lets the root complex
or switch direct DMA to a specific cache slice / NUMA node, cutting
cross-socket snoop traffic and DRAM pressure under sustained p2p
workloads.

Until now the mlx5 importer had no way to learn the exporter's chosen
ST tag, so dma-buf MRs were registered without TPH and ran with the
default (no-hint) routing. With dma_buf_get_pci_tph() in place this
patch wires up mlx5_ib to query that metadata at MR registration time
for p2p access and use it to program requester-side TPH on the outbound
mkey. If the exporter has no metadata, fall back to the existing
no-TPH path so behavior for non-TPH-aware exporters is unchanged.

Use mlx5_st_alloc_index_by_tag() to translate exporter-provided
steering tags into local ST entries when table mode is active, and add
mlx5_st_get_index() for DMAH-backed flows that already carry an ST
index.

Splitting mlx5_st_alloc_index() moves the !dev->st check into the new
by-tag helper, so keep an equivalent early-out in mlx5_st_alloc_index()
itself, ahead of the pcie_tph_get_cpu_st() call. A device without ST
support therefore still returns -EOPNOTSUPP without an ACPI _DSM
invocation, exactly as it did before the split. The check is repeated in
mlx5_st_alloc_index_by_tag() because that is now an exported entry point
in its own right.

For TPH-backed FRMRs, keep the extra ST-table reference tied to MR
lifetime rather than pooled mkey lifetime. Acquire the ref before MR
creation and release it again when the MR is returned to the pool or
the backing mkey is destroyed, while leaving the generic FRMR pool
core unchanged.

The exporter's steering tag is only valid for the mapping it was queried
against (see dma_buf_get_pci_tph()). The mkey's TPH fields cannot be
reprogrammed in place: they are create-time fields with no UMR update
mask, and rebuilding the mkey would change the rkey/lkey, which must stay
stable across invalidate/refault.

Keep the registration-time TPH tuple in the MR and re-query the exporter
after each dma-buf mapping is established under dmabuf->resv. If the
tuple is unchanged, continue using the existing mkey. If the tuple changed
or disappeared, unmap the pages and return -EFAULT rather than continue
with a stale mkey hint; userspace must create a new MR to get a mkey
programmed with the new tuple. For vfio-pci BAR dma-bufs this validation
is expected to be a no-op because invalidation is revoke/quiesce of BAR
access, not movement to a different backing placement, and the
userspace-provided TPH tuple is not changed by the revoke/un-revoke path.

Import the DMA_BUF namespace for the new dma_buf_get_pci_tph() call so
modular mlx5_ib builds link cleanly.

Signed-off-by: Zhiping Zhang <redacted>
---
 drivers/infiniband/hw/mlx5/main.c             |   1 +
 drivers/infiniband/hw/mlx5/mlx5_ib.h          |  11 ++
 drivers/infiniband/hw/mlx5/mr.c               | 151 +++++++++++++++++-
 drivers/infiniband/hw/mlx5/odp.c              |   7 +
 .../net/ethernet/mellanox/mlx5/core/lib/st.c  |  52 +++++-
 include/linux/mlx5/driver.h                   |  15 ++
 6 files changed, 223 insertions(+), 14 deletions(-)

diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 02809114fc79..a2b497f6b16b 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c

@@ -60,6 +60,7 @@
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
 MODULE_LICENSE("Dual BSD/GPL");
+MODULE_IMPORT_NS("DMA_BUF");

 struct mlx5_ib_event_work {
        struct work_struct      work;

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 522984d958bb..c6a956b2ee87 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h

@@ -646,6 +646,15 @@ enum mlx5_mkey_type {
 /* Used for non-existent ph value */
 #define MLX5_IB_NO_PH 0xff

+struct dma_buf;
+
+struct mlx5_ib_dmabuf_tph {
+       u16 steering_tag;
+       u8 ph;
+       bool extended;
+       bool valid;
+};
+
 struct mlx5_ib_mkey {
        u32 key;
        enum mlx5_mkey_type type;

@@ -726,6 +735,7 @@ struct mlx5_ib_mr {
                        u8 revoked :1;
                        /* Indicates previous dmabuf page fault occurred */
                        u8 dmabuf_faulted:1;
+                       struct mlx5_ib_dmabuf_tph dmabuf_tph;
                        struct mlx5_ib_mkey null_mmkey;
                };
        };

@@ -1341,6 +1351,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
                                    u64 length, u64 virt_addr, int access_flags,
                                    struct ib_pd *pd, struct ib_udata *udata);
 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
+int mlx5_ib_validate_dmabuf_tph(struct mlx5_ib_mr *mr, struct dma_buf *dmabuf);
 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
                               u32 max_num_sg);
 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,

diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index e6b74955d95d..404c5d081b8a 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c

@@ -39,6 +39,7 @@
 #include <linux/delay.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-resv.h>
+#include <linux/pci-tph.h>
 #include <rdma/frmr_pools.h>
 #include <rdma/ib_umem_odp.h>
 #include "dm.h"

@@ -167,6 +168,61 @@ static int get_unchangeable_access_flags(struct mlx5_ib_dev *dev,
 #define MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK GENMASK_ULL(23, 16)
 #define MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK GENMASK_ULL(15, 0)

+static u8 mlx5_ib_tph_key_ph(u8 ph)
+{
+       if (ph == MLX5_IB_NO_PH || ph == 0)
+               ph ^= MLX5_IB_NO_PH;
+
+       return ph;
+}
+
+static int mlx5_ib_get_st_handle_ref(struct mlx5_ib_dev *dev, u16 st_index)
+{
+       if (st_index == MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)
+               return 0;
+
+       return mlx5_st_get_index(dev->mdev, st_index);
+}
+
+static void mlx5_ib_put_st_index_ref(struct mlx5_ib_dev *dev, u16 st_index)
+{
+       if (st_index == MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)
+               return;
+
+       mlx5_st_dealloc_index(dev->mdev, st_index);
+}
+
+static void mlx5_ib_put_st_handle_ref(struct mlx5_ib_dev *dev,
+                                     u64 kernel_vendor_key)
+{
+       u16 st_index = FIELD_GET(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK,
+                                kernel_vendor_key);
+
+       mlx5_ib_put_st_index_ref(dev, st_index);
+}
+
+int mlx5_ib_validate_dmabuf_tph(struct mlx5_ib_mr *mr, struct dma_buf *dmabuf)
+{
+       struct mlx5_ib_dmabuf_tph *tph = &mr->dmabuf_tph;
+       u16 steering_tag;
+       u8 ph;
+       int ret;
+
+       if (!tph->valid)
+               return 0;
+
+       dma_resv_assert_held(dmabuf->resv);
+
+       ret = dma_buf_get_pci_tph(dmabuf, tph->extended, &steering_tag, &ph);
+       if (ret)
+               return -EFAULT;
+
+       if (steering_tag != tph->steering_tag || ph != tph->ph)
+               return -EFAULT;
+
+       return 0;
+}
+
 static struct mlx5_ib_mr *
 _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem,
                      int access_flags, int access_mode,

@@ -189,13 +245,10 @@ _mlx5_frmr_pool_alloc(struct mlx5_ib_dev *dev, struct ib_umem *umem,
                        MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK :
                        0;

-       /* Normalize ph: swap 0 and MLX5_IB_NO_PH */
-       if (ph == MLX5_IB_NO_PH || ph == 0)
-               ph ^= MLX5_IB_NO_PH;
-
        mr->ibmr.frmr.key.kernel_vendor_key =
                FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) |
-               FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK, ph);
+               FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK,
+                          mlx5_ib_tph_key_ph(ph));
        err = ib_frmr_pool_pop(&dev->ib_dev, &mr->ibmr);
        if (err) {
                kfree(mr);

@@ -218,7 +271,9 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
                                      0 :
                                      MLX5_FRMR_POOLS_KEY_ACCESS_MODE_KSM_MASK,
                .num_dma_blocks = ndescs,
-               .kernel_vendor_key = 0, /* no PH and no ST index */
+               .kernel_vendor_key =
+                       FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK,
+                                  MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX),
        };
        struct mlx5_ib_mr *mr;
        int ret;

@@ -557,6 +612,10 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
        mr->ibmr.pd = pd;
        mr->access_flags = access_flags;
        mr->page_shift = order_base_2(page_size);
+       mr->ibmr.frmr.key.kernel_vendor_key =
+               FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_ST_INDEX_MASK, st_index) |
+               FIELD_PREP(MLX5_FRMR_POOLS_KERNEL_KEY_PH_MASK,
+                          mlx5_ib_tph_key_ph(ph));

        inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
        if (populate)

@@ -753,6 +812,12 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem,
                        st_index = mdmah->st_index;
        }

+       err = mlx5_ib_get_st_handle_ref(dev, st_index);
+       if (err) {
+               ib_umem_release(umem);
+               return ERR_PTR(err);
+       }
+
        xlt_with_umr = mlx5r_umr_can_load_pas(dev, umem->length);
        if (xlt_with_umr) {
                mr = alloc_cacheable_mr(pd, umem, iova, access_flags,

@@ -769,6 +834,7 @@ static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem,
                mutex_unlock(&dev->slow_path_mutex);
        }
        if (IS_ERR(mr)) {
+               mlx5_ib_put_st_index_ref(dev, st_index);
                ib_umem_release(umem);
                return ERR_CAST(mr);
        }

@@ -903,6 +969,63 @@ static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = {
        .invalidate_mappings = mlx5_ib_dmabuf_invalidate_cb,
 };

+static void get_pci_tph_mr_dmabuf(struct mlx5_ib_dev *dev, struct dma_buf *dmabuf,
+                                 struct mlx5_ib_dmabuf_tph *tph,
+                                 u16 *st_index, u8 *ph)
+{
+       u16 local_st_index;
+       u16 steering_tag;
+       u8 local_ph;
+       bool extended;
+       int ret;
+
+       switch (pcie_tph_enabled_req_type(dev->mdev->pdev)) {
+       case PCI_TPH_REQ_TPH_ONLY:
+               extended = false;
+               break;
+       case PCI_TPH_REQ_EXT_TPH:
+               extended = true;
+               break;
+       default:
+               return;
+       }
+
+       /*
+        * dma_buf_get_pci_tph() must run under dmabuf->resv, but drop the lock
+        * before mlx5_st_alloc_index_by_tag(): that allocates with GFP_KERNEL,
+        * which must not run under dmabuf->resv since reclaim can re-enter the
+        * same reservation object through a dynamic exporter's move_notify path.
+        */
+       dma_resv_lock(dmabuf->resv, NULL);
+       ret = dma_buf_get_pci_tph(dmabuf, extended, &steering_tag, &local_ph);
+       dma_resv_unlock(dmabuf->resv);
+       if (ret) {
+               if (ret != -EOPNOTSUPP)
+                       mlx5_ib_dbg(dev, "get_pci_tph failed (%d)\n", ret);
+               return;
+       }
+
+       ret = mlx5_st_alloc_index_by_tag(dev->mdev, steering_tag,
+                                        &local_st_index);
+       if (ret) {
+               mlx5_ib_dbg(dev, "st_alloc_index_by_tag failed (%d)\n", ret);
+               return;
+       }
+
+       *st_index = local_st_index;
+       *ph = local_ph;
+       tph->steering_tag = steering_tag;
+       tph->ph = local_ph;
+       tph->extended = extended;
+       tph->valid = true;
+}
+
+static void mlx5_ib_mr_put_st_handle_ref(struct mlx5_ib_mr *mr)
+{
+       mlx5_ib_put_st_handle_ref(mr_to_mdev(mr),
+                                 mr->ibmr.frmr.key.kernel_vendor_key);
+}
+
 static struct ib_mr *
 reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
                   u64 offset, u64 length, u64 virt_addr,

@@ -913,6 +1036,7 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
        struct mlx5_ib_dev *dev = to_mdev(pd->device);
        struct mlx5_ib_mr *mr = NULL;
        struct ib_umem_dmabuf *umem_dmabuf;
+       struct mlx5_ib_dmabuf_tph tph = {};
        u16 st_index = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX;
        u8 ph = MLX5_IB_NO_PH;
        int err;

@@ -945,18 +1069,29 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
                ph = dmah->ph;
                if (dmah->valid_fields & BIT(IB_DMAH_CPU_ID_EXISTS))
                        st_index = mdmah->st_index;
+
+               err = mlx5_ib_get_st_handle_ref(dev, st_index);
+               if (err) {
+                       ib_umem_release(&umem_dmabuf->umem);
+                       return ERR_PTR(err);
+               }
+       } else {
+               get_pci_tph_mr_dmabuf(dev, umem_dmabuf->attach->dmabuf,
+                                     &tph, &st_index, &ph);
        }

        mr = alloc_cacheable_mr(pd, &umem_dmabuf->umem, virt_addr,
                                access_flags, access_mode,
                                st_index, ph);
        if (IS_ERR(mr)) {
+               mlx5_ib_put_st_index_ref(dev, st_index);
                ib_umem_release(&umem_dmabuf->umem);
                return ERR_CAST(mr);
        }

        mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);

+       mr->dmabuf_tph = tph;
        atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages);
        umem_dmabuf->private = mr;
        if (!pinned_mode) {

@@ -1405,6 +1540,7 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr)
        if (mr->ibmr.frmr.pool) {
                if (!mlx5_umr_revoke_mr_with_lock(mr)) {
                        ib_frmr_pool_push(mr->ibmr.device, &mr->ibmr);
+                       mlx5_ib_mr_put_st_handle_ref(mr);
                        return 0;
                }
        }

@@ -1432,6 +1568,9 @@ static int mlx5r_handle_mkey_cleanup(struct mlx5_ib_mr *mr)
        if (mr->ibmr.frmr.pool && !ret)
                ib_frmr_pool_drop(&mr->ibmr);

+       if (!ret)
+               mlx5_ib_mr_put_st_handle_ref(mr);
+
        return ret;
 }

diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 1badec9bf527..7c1731aa3dd9 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c

@@ -857,6 +857,13 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt,
                return err;
        }

+       err = mlx5_ib_validate_dmabuf_tph(mr, umem_dmabuf->attach->dmabuf);
+       if (err) {
+               ib_umem_dmabuf_unmap_pages(umem_dmabuf);
+               dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
+               return err;
+       }
+
        page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode);
        if (!page_size) {
                ib_umem_dmabuf_unmap_pages(umem_dmabuf);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c
index 7cedc348790d..c62bcae7c542 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/st.c

@@ -92,23 +92,18 @@ void mlx5_st_destroy(struct mlx5_core_dev *dev)
        kfree(st);
 }

-int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type,
-                       unsigned int cpu_uid, u16 *st_index)
+int mlx5_st_alloc_index_by_tag(struct mlx5_core_dev *dev, u16 tag,
+                              u16 *st_index)
 {
        struct mlx5_st_idx_data *idx_data;
        struct mlx5_st *st = dev->st;
        unsigned long index;
        u32 xa_id;
-       u16 tag;
-       int ret;
+       int ret = 0;

        if (!st)
                return -EOPNOTSUPP;

-       ret = pcie_tph_get_cpu_st(dev->pdev, mem_type, cpu_uid, &tag);
-       if (ret)
-               return ret;
-
        if (st->direct_mode) {
                *st_index = tag;
                return 0;

@@ -152,8 +147,49 @@ int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type,
        mutex_unlock(&st->lock);
        return ret;
 }
+EXPORT_SYMBOL_GPL(mlx5_st_alloc_index_by_tag);
+
+int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type,
+                       unsigned int cpu_uid, u16 *st_index)
+{
+       u16 tag;
+       int ret;
+
+       if (!dev->st)
+               return -EOPNOTSUPP;
+
+       ret = pcie_tph_get_cpu_st(dev->pdev, mem_type, cpu_uid, &tag);
+       if (ret)
+               return ret;
+
+       return mlx5_st_alloc_index_by_tag(dev, tag, st_index);
+}
 EXPORT_SYMBOL_GPL(mlx5_st_alloc_index);

+int mlx5_st_get_index(struct mlx5_core_dev *dev, u16 st_index)
+{
+       struct mlx5_st_idx_data *idx_data;
+       struct mlx5_st *st = dev->st;
+       int ret = 0;
+
+       if (!st)
+               return -EOPNOTSUPP;
+
+       if (st->direct_mode)
+               return 0;
+
+       mutex_lock(&st->lock);
+       idx_data = xa_load(&st->idx_xa, st_index);
+       if (WARN_ON_ONCE(!idx_data))
+               ret = -EINVAL;
+       else
+               refcount_inc(&idx_data->usecount);
+       mutex_unlock(&st->lock);
+
+       return ret;
+}
+EXPORT_SYMBOL_GPL(mlx5_st_get_index);
+
 int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index)
 {
        struct mlx5_st_idx_data *idx_data;

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index b1871c0821d0..d26e5744c843 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h

@@ -1182,16 +1182,31 @@ int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type
                           u64 length, u16 uid, phys_addr_t addr, u32 obj_id);

 #ifdef CONFIG_PCIE_TPH
+int mlx5_st_alloc_index_by_tag(struct mlx5_core_dev *dev, u16 tag,
+                              u16 *st_index);
+int mlx5_st_get_index(struct mlx5_core_dev *dev, u16 st_index);
 int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type,
                        unsigned int cpu_uid, u16 *st_index);
 int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index);
 #else
+static inline int mlx5_st_alloc_index_by_tag(struct mlx5_core_dev *dev,
+                                            u16 tag, u16 *st_index)
+{
+       return -EOPNOTSUPP;
+}
+
+static inline int mlx5_st_get_index(struct mlx5_core_dev *dev, u16 st_index)
+{
+       return -EOPNOTSUPP;
+}
+
 static inline int mlx5_st_alloc_index(struct mlx5_core_dev *dev,
                                      enum tph_mem_type mem_type,
                                      unsigned int cpu_uid, u16 *st_index)
 {
        return -EOPNOTSUPP;
 }
+
 static inline int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index)
 {
        return -EOPNOTSUPP;
--

2.53.0-Meta