[PATCH 17/23] net: hpe: ss1: add the datapath and memory translation layers

@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2019, 2024-2026 Hewlett Packard Enterprise Development LP */
+
+/*
+ * Address Translation Unit (ATU) management
+ * ATS functionality
+ */
+
+#include <linux/pci.h>
+#include <linux/types.h>
+#include <linux/iommu.h>
+
+#include "cass_core.h"
+
+/* PCIe IP PF0_PORT_LOGIC register */
+#define SYMBOL_TIMER_FILTER_1_OFF 0x71c
+#define CX_FLT_MASK_CPL_LEN_MATCH (1 << 26)
+#define FILTER_MASK_2_OFF 0x720
+#define CX_FLT_UNMASK_ATS_SPECIFIC_RULES (1 << 10)
+
+#define ATU_NUM_PASIDS ATU_PHYS_AC
+#define MAX_PAGE_REQS 512
+
+#ifdef CONFIG_IOMMU_SVA
+
+static bool enable_ats = true;
+module_param(enable_ats, bool, 0644);
+MODULE_PARM_DESC(enable_ats, "Enable Address Translation Services (PCIe ATS)");
+
+static bool odp_mode_nic_pri;
+module_param(odp_mode_nic_pri, bool, 0644);
+MODULE_PARM_DESC(odp_mode_nic_pri,
+		 "ODP mode used for ATS. ATS_PRS:0, NIC_PRI:1");
+
+static int cass_bind_sva(struct pci_dev *pdev, struct cass_ac *cac)
+{
+	u32 pasid;
+	struct iommu_sva *sva;
+
+	sva = iommu_sva_bind_device(&pdev->dev, current->mm);
+	if (IS_ERR(sva)) {
+		pr_err("iommu_sva_bind_device failed ret:%ld\n", PTR_ERR(sva));
+		return PTR_ERR(sva);
+	}
+
+	pasid = iommu_sva_get_pasid(sva);
+	if (pasid == IOMMU_PASID_INVALID) {
+		pr_err("pasid invalid\n");
+		iommu_sva_unbind_device(sva);
+		return -ENODEV;
+	}
+
+	cac->sva = sva;
+	cac->pasid = pasid;
+
+	return 0;
+}
+
+static void cass_unbind_sva(const struct cass_ac *cac)
+{
+	iommu_sva_unbind_device(cac->sva);
+}
+
+/**
+ * cass_bind_ac() - Bind an address context to the current task.
+ *
+ * @hw:  Cassini device
+ * @cac: Cassini AC struct
+ *
+ * @return: 0 on success
+ */
+static int cass_bind_ac(struct cass_dev *hw, struct cass_ac *cac)
+{
+	struct pci_dev *pdev = hw->cdev.pdev;
+
+	if (!pdev->ats_enabled) {
+		cxidev_dbg(&hw->cdev, "ATS not enabled\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (!pdev->pasid_enabled) {
+		cxidev_dbg(&hw->cdev, "PASID not enabled\n");
+		return -EOPNOTSUPP;
+	}
+
+	return cass_bind_sva(pdev, cac);
+}
+
+/**
+ * cass_unbind_ac() - Unbind an address context. Called during AC cleanup.
+ *
+ * @hw:  Cassini device
+ * @cac: Cassini AC struct
+ */
+void cass_unbind_ac(struct cass_dev *hw, const struct cass_ac *cac)
+{
+	if (!enable_ats)
+		return;
+
+	if (cac->sva)
+		cass_unbind_sva(cac);
+}
+
+void cass_iommu_fini(struct cass_dev *hw)
+{
+}
+
+/**
+ * cass_iommu_init() - Initialize the AMD/Intel IOMMU for the physical device
+ *
+ * If the IOMMU is not enabled or available, ATS mode is not supported.
+ *
+ * @hw: Cassini device
+ */
+void cass_iommu_init(struct cass_dev *hw)
+{
+	int pos;
+	u32 filter_mask;
+	u32 max_requests;
+	struct pci_dev *pdev = hw->cdev.pdev;
+
+	/*
+	 * Mask length match for completions.
+	 * The PCI interface checks if the completion length matches
+	 * the requested length. Unfortunately, this is not the case for
+	 * ATS transactions as only pages that are present are returned
+	 * so enable Mask length match for Cassini 1.
+	 */
+	if (cass_version(hw, CASSINI_1)) {
+		pci_read_config_dword(pdev, SYMBOL_TIMER_FILTER_1_OFF,
+				      &filter_mask);
+		filter_mask |= CX_FLT_MASK_CPL_LEN_MATCH;
+		pci_write_config_dword(pdev, SYMBOL_TIMER_FILTER_1_OFF,
+				       filter_mask);
+	}
+
+	/*
+	 * Enable CX_FLT_UNMASK_ATS_SPECIFIC_RULES for C2.
+	 *
+	 * Cassini 2 has a new capability which uses
+	 * CX_FLT_UNMASK_ATS_SPECIFIC_RULES. The PCI controller used by
+	 * C2 has a reference section which states the following:
+	 * Lower Address is not checked for Cpls related to ATS Requests.
+	 * An ATS-related Cpl completes the request if it has a Byte Count
+	 * that is equal to four times the Length field.
+	 */
+	if (cass_version(hw, CASSINI_2)) {
+		pci_read_config_dword(pdev, FILTER_MASK_2_OFF, &filter_mask);
+		filter_mask |= CX_FLT_UNMASK_ATS_SPECIFIC_RULES;
+		pci_write_config_dword(pdev, FILTER_MASK_2_OFF, filter_mask);
+	}
+
+	/*
+	 * The AMD IOMMU is currently hardcoding max requests to 32.
+	 * Set to MAX_PAGE_REQS.
+	 */
+	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
+	if (!pos) {
+		dev_warn(&pdev->dev, "Error: PRI extended capability in PCI config not present\n");
+		return;
+	}
+
+	pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests);
+	max_requests = min_t(u32, max_requests, (u32)MAX_PAGE_REQS);
+	pdev->pri_reqs_alloc = max_requests;
+	pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, max_requests);
+}
+
+/**
+ * cass_ats_init() - Initialize an AC using either ATS or ATS passthrough
+ *                   and bind the AC to the current task
+ *
+ * @lni_priv: Private LNI struct
+ * @m_opts: User map options
+ * @cac:    Cassini AC struct
+ *
+ * @return: 0 on success, negative value on failure
+ */
+int cass_ats_init(struct cxi_lni_priv *lni_priv,
+		  struct ac_map_opts *m_opts,
+		  struct cass_ac *cac)
+{
+	int ret;
+	bool privileged = !(m_opts->flags & CXI_MAP_USER_ADDR);
+	union c_atu_cfg_ac_table *ac = &cac->cfg_ac;
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	bool have_pri = hw->cdev.pdev->pri_enabled;
+
+	if (!enable_ats)
+		return -EOPNOTSUPP;
+
+	ret = cass_bind_ac(hw, cac);
+	if (ret)
+		return ret;
+
+	if (m_opts->flags & CXI_MAP_ATS_PT)
+		ac->ta_mode = C_ATU_PASSTHROUGH_MODE;
+	else if (m_opts->flags & CXI_MAP_ATS_DYN)
+		ac->ta_mode = C_ATU_ATS_DYNAMIC_MODE;
+	else
+		ac->ta_mode = C_ATU_ATS_MODE;
+
+	m_opts->iova = m_opts->va_start;
+	cac->iova_base = 0;
+	cac->iova_len = m_opts->flags & CXI_MAP_USER_ADDR ?
+				TASK_SIZE : ULONG_MAX;
+	cac->iova_end = cac->iova_base + cac->iova_len - 1;
+
+	ac->cntr_pool_id = 1;
+	ac->ats_vf_en = lni_priv->is_vf;
+	ac->ats_vf_num = lni_priv->is_vf ? lni_priv->vf_num : 0;
+	ac->ats_no_write = 0;
+	ac->ats_pasid = cac->pasid;
+	ac->ats_pasid_en = 1;
+	ac->ats_pasid_er = 0;
+	ac->ats_pasid_pmr = !!privileged;
+	ac->mem_base = cac->iova_base >> ATU_CFG_AC_TABLE_MB_SHIFT;
+	ac->mem_size = cac->iova_len >> C_ADDR_SHIFT;
+	ac->odp_mode = have_pri && !odp_mode_nic_pri ?
+			C_ATU_ODP_MODE_ATS_PRS : C_ATU_ODP_MODE_NIC_PRI;
+
+	return 0;
+}
+
+/**
+ * cass_ats_md_init() - Pin ATS pages or set up notifier
+ *
+ * When using ODP with NIC_PRI mode, the mmu notifier will be initialized
+ * for this descriptor. If pinning, just pin the pages.
+ *
+ * @md_priv: Private memory descriptor
+ * @m_opts:  User options containing the IOVA range
+ *
+ * @return: Success or -ENOMEM
+ */
+int cass_ats_md_init(struct cxi_md_priv *md_priv,
+		     const struct ac_map_opts *m_opts)
+{
+	int ret;
+	int write = m_opts->flags & CXI_MAP_WRITE;
+	int npages = md_priv->md.len >> md_priv->md.page_shift;
+	struct page **pages;
+	size_t size = npages * sizeof(*pages);
+	struct cass_dev *hw = container_of(md_priv->lni_priv->dev,
+					   struct cass_dev, cdev);
+
+	if (!enable_ats)
+		return -EOPNOTSUPP;
+
+	if (!(m_opts->flags & CXI_MAP_PIN)) {
+		if (odp_mode_nic_pri || !hw->cdev.pdev->pri_enabled)
+			return cass_mmu_notifier_insert(md_priv, m_opts);
+
+		return 0;
+	}
+
+	pages = kvmalloc(size, GFP_KERNEL);
+	if (!pages)
+		return -ENOMEM;
+
+	ret = cass_pin(md_priv->cac, pages, npages, md_priv->md.va, write);
+
+	if (ret) {
+		kvfree(pages);
+		return ret;
+	}
+
+	md_priv->pages = pages;
+
+	return ret;
+}
+
+#else /* CONFIG_IOMMU_SVA */
+void cass_unbind_ac(struct cass_dev *hw, const struct cass_ac *cac)
+{
+}
+
+void cass_iommu_fini(struct cass_dev *hw)
+{
+}
+
+void cass_iommu_init(struct cass_dev *hw)
+{
+}
+
+int cass_ats_init(struct cxi_lni_priv *lni_priv,
+		  struct ac_map_opts *m_opts,
+		  struct cass_ac *cac)
+{
+	return -ENODEV;
+}
+
+int cass_ats_md_init(struct cxi_md_priv *md_priv,
+		     const struct ac_map_opts *m_opts)
+{
+	return -ENODEV;
+}
+#endif /* !CONFIG_IOMMU_SVA */

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2022 Hewlett Packard Enterprise Development LP */
+
+#ifdef CONFIG_AMD_IOMMU
+
+#include <linux/module.h>
+#include <linux/amd-iommu.h>
+
+#include "cass_core.h"
+
+/*
+ * ATS invalidate interface for Cassini 1.1
+ */
+
+typedef void (*amd_iommu_invalidate_range)(struct pci_dev *pdev,
+					   int pasid, unsigned long start,
+					   unsigned long end);
+int amd_iommu_set_invalidate_range_cb(struct pci_dev *pdev,
+				      amd_iommu_invalidate_range cb);
+
+/**
+ * cass_ats_invalidate_range_cb() - Callback from the AMD IOMMU to invalidate
+ *				    and address range.
+ *
+ * @pdev: PCI device
+ * @pasid: The process address space ID.
+ * @start: Start of address range
+ * @end: End of address range
+ */
+static void cass_ats_invalidate_range_cb(struct pci_dev *pdev, int pasid,
+					 unsigned long start, unsigned long end)
+{
+
+	struct cass_dev *hw = pci_get_drvdata(pdev);
+	struct cass_ac *cac;
+
+	cac = cass_cac_get(hw, pasid);
+	if (!cac)
+		return;
+
+	pr_debug("pasid:%d start:%lx end:%lx\n", pasid, start, end);
+
+	cass_invalidate_range(cac, start, end - start);
+	atomic_inc(&hw->dcpl_ats_mn_inval);
+	cass_cac_put(cac);
+}
+
+/**
+ * cass_amd_iommu_inval_cb_init() - Initialize the invalidation callback.
+ *
+ * @pdev: PCI device
+ */
+void cass_amd_iommu_inval_cb_init(struct pci_dev *pdev)
+{
+	int (*set_inval_range_cb)(struct pci_dev *pdev,
+				  amd_iommu_invalidate_range cb);
+	struct cass_dev *hw = pci_get_drvdata(pdev);
+	int ret;
+
+	/* Only set up invalidation callback for Cassini 1 */
+	if (!cass_version(hw, CASSINI_1))
+		return;
+
+	set_inval_range_cb = symbol_get(amd_iommu_set_invalidate_range_cb);
+	if (set_inval_range_cb) {
+		ret = set_inval_range_cb(pdev, cass_ats_invalidate_range_cb);
+		if (ret) {
+			dev_WARN(&pdev->dev,
+				 "Error setting invalidation callback (%d)\n",
+				 ret);
+		} else {
+			hw->ats_c1_odp_enable = true;
+		}
+
+		symbol_put(amd_iommu_set_invalidate_range_cb);
+	}
+}
+
+#endif /* CONFIG_AMD_IOMMU */

@@ -0,0 +1,2072 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2018, 2024-2026 Hewlett Packard Enterprise Development LP */
+
+/* Address Translation Unit (ATU) management */
+
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <linux/hugetlb.h>
+#include <linux/iova.h>
+#include <linux/bvec.h>
+#include <linux/iopoll.h>
+#include <linux/workqueue.h>
+#include <linux/random.h>
+
+#include "cass_core.h"
+#include "cass_ss1_debugfs.h"
+
+int more_debug;
+module_param(more_debug, int, 0644);
+MODULE_PARM_DESC(more_debug, "More debug");
+
+static int default_ptg_mode = C_ATU_PTG_MODE_DBL_C;
+module_param(default_ptg_mode, int, 0644);
+MODULE_PARM_DESC(default_ptg_mode, "Default PTG Mode");
+
+/* Invalidation interval is 2^(inval_process_interval+1) - 10 is ~2 uS */
+static int inval_process_interval = 10;
+module_param(inval_process_interval, int, 0644);
+MODULE_PARM_DESC(inval_process_interval, "ATS invalidate processing interval");
+
+/* Relaxed ordering bits test */
+static bool acxt_mem_rd_ro;
+module_param(acxt_mem_rd_ro, bool, 0644);
+MODULE_PARM_DESC(acxt_mem_rd_ro, "Memory Read Relaxed Ordering");
+static bool acxt_flush_req_ro;
+module_param(acxt_flush_req_ro, bool, 0644);
+MODULE_PARM_DESC(acxt_flush_req_ro, "Flush Request Relaxed Ordering");
+static bool acxt_translation_req_ro;
+module_param(acxt_translation_req_ro, bool, 0644);
+MODULE_PARM_DESC(acxt_translation_req_ro, "Translation Request Relaxed Ordering");
+static bool acxt_fetching_amo_ro;
+module_param(acxt_fetching_amo_ro, bool, 0644);
+MODULE_PARM_DESC(acxt_fetching_amo_ro, "Fetching AMO Relaxed Ordering");
+static bool acxt_ido;
+module_param(acxt_ido, bool, 0644);
+MODULE_PARM_DESC(acxt_ido, "ID-Based Ordering");
+
+static bool ac_salt = true;
+module_param(ac_salt, bool, 0644);
+MODULE_PARM_DESC(ac_salt, "Add salt to ACs");
+
+static bool acxt_ph_en;
+module_param(acxt_ph_en, bool, 0644);
+MODULE_PARM_DESC(acxt_ph_en, "Enable use of Processing Hints and Steering Tag");
+static u8 acxt_phints;
+module_param(acxt_phints, byte, 0644);
+MODULE_PARM_DESC(acxt_phints, "TLP Processing Hints");
+static u8 acxt_steering_tag;
+module_param(acxt_steering_tag, byte, 0644);
+MODULE_PARM_DESC(acxt_steering_tag, "Steering Tag");
+
+bool ats_c1_override;
+module_param(ats_c1_override, bool, 0644);
+MODULE_PARM_DESC(ats_c1_override, "Override ODP Enable for ATS with C1");
+
+/* Use a larger base page size when huge_shift - page_shift is greater
+ * than the maximum page table size (max-threshold) allowed. This allows
+ * us to support 256, 512 and 1024 MB Cray hugepage sizes.
+ */
+static int hugepage_threshold = 1;
+module_param(hugepage_threshold, int, 0644);
+MODULE_PARM_DESC(hugepage_threshold, "Threshold to increase base page size.");
+
+void cass_atu_fini(struct cass_dev *hw)
+{
+	cxi_unregister_hw_errors(hw, &hw->atu_err);
+	iova_cache_put();
+	dma_unmap_single(&hw->cdev.pdev->dev, hw->oxe_dummy_dma_addr,
+			 PAGE_SIZE, DMA_TO_DEVICE);
+	kfree(hw->oxe_dummy_addr);
+	ida_destroy(&hw->atu_table);
+	cass_iommu_fini(hw);
+	cass_nta_pri_fini(hw);
+	cass_nta_cq_fini(hw);
+	refcount_dec(&hw->refcount);
+}
+
+/* Atomically write an Address Context */
+static void cass_write_ac(struct cass_dev *hw, union c_atu_cfg_ac_table *ac,
+			  unsigned int index, bool user, bool is_physac,
+			  bool vf_en, int vf_num)
+{
+	union c_pi_cfg_acxt pi_cfg_acxt = {
+		.ac_en = ac->context_en,
+	};
+
+	pi_cfg_acxt.vf_en = vf_en;
+	pi_cfg_acxt.vf_num = vf_num;
+
+	if (ac->ta_mode != C_ATU_NTA_MODE) {
+		pi_cfg_acxt.pasid = ac->ats_pasid;
+		pi_cfg_acxt.pasid_en = ac->ats_pasid_en;
+		pi_cfg_acxt.pasid_er = ac->ats_pasid_er;
+		pi_cfg_acxt.pasid_pmr = ac->ats_pasid_pmr;
+	}
+
+	pi_cfg_acxt.mem_rd_ro = acxt_mem_rd_ro;
+	pi_cfg_acxt.flush_req_ro = acxt_flush_req_ro;
+	pi_cfg_acxt.translation_req_ro = acxt_translation_req_ro;
+	pi_cfg_acxt.fetching_amo_ro = acxt_fetching_amo_ro;
+	pi_cfg_acxt.ido = acxt_ido;
+
+	if (acxt_ph_en && user) {
+		pi_cfg_acxt.ph_en = acxt_ph_en;
+		pi_cfg_acxt.phints = acxt_phints;
+		pi_cfg_acxt.steering_tag = acxt_steering_tag;
+	}
+
+	if (ac_salt) {
+		u32 rand = get_random_u32();
+
+		ac->ptn_salt = rand;
+		ac->idx_salt = rand >> 2;
+	}
+
+	spin_lock(&hw->atu_shadow_lock);
+	cass_config_ac(hw, index, ac);
+	spin_unlock(&hw->atu_shadow_lock);
+
+	/* WORKAROUND: Cassini ERRATA-3277
+	 * When disabling an AC, invalidate before disabling the ACXT.
+	 */
+	if (!is_physac && cass_version(hw, CASSINI_1) && !ac->context_en) {
+		struct cass_ac *cac = container_of(ac, struct cass_ac, cfg_ac);
+
+		cass_invalidate_range(cac, 0, ATUCQ_INVALIDATE_ALL);
+	}
+
+	cass_config_pi_acxt(hw, index, &pi_cfg_acxt);
+}
+
+/* Callback registered with the interrupt error handler, executed when
+ * C_ATU_ERR_FLG.PRB_EXPIRED error interrupt is detected.
+ */
+static void cass_prb_exp_cb(struct cass_dev *hw, unsigned int irq,
+			    bool is_ext, unsigned int bitn)
+{
+	queue_work(hw->prb_wq, &hw->prb_work);
+}
+
+int cass_atu_init(struct cass_dev *hw)
+{
+	int i;
+	int ret;
+	union c_atu_cfg_nta cfg_nta = {
+		.acid = ATU_NTA_DEF_PI_ACXT,
+	};
+	union c_atu_cfg_ac_table ac = {
+		.context_en = 1,
+		.ta_mode = C_ATU_PASSTHROUGH_MODE,
+	};
+	union c_oxe_cfg_dummy_addr oxe_dummy = {
+		.ac = ATU_PHYS_AC
+	};
+	union c_atu_cfg_ats cfg_ats;
+
+	ac.mem_size = ~ac.mem_size;
+
+	hw->oxe_dummy_addr = kcalloc(1, PAGE_SIZE, GFP_KERNEL);
+	if (hw->oxe_dummy_addr == NULL)
+		return -ENOMEM;
+
+	ida_init(&hw->atu_table);
+	refcount_inc(&hw->refcount);
+	spin_lock_init(&hw->cac_table_lock);
+	spin_lock_init(&hw->atu_shadow_lock);
+	mutex_init(&hw->odpq_mutex);
+	atomic_set(&hw->stats.md, 0);
+	atomic_set(&hw->atu_error_inject, 0);
+	atomic_set(&hw->atu_odp_requests, 0);
+	atomic_set(&hw->atu_odp_fails, 0);
+	atomic_set(&hw->atu_prb_expired, 0);
+	atomic_set(&hw->dcpl_md_clear_inval, 0);
+	atomic_set(&hw->dcpl_nta_mn_inval, 0);
+	atomic_set(&hw->dcpl_ats_mn_inval, 0);
+	atomic_set(&hw->dcpl_comp_wait, 0);
+	atomic_set(&hw->dcpl_entered, 0);
+	atomic_set(&hw->dcpl_ixe_cntr_0, 0);
+	atomic_set(&hw->dcpl_ixe_cntr_dec_0, 0);
+	atomic_set(&hw->dcpl_ee_cntr_dec_0, 0);
+	atomic_set(&hw->dcpl_ee_cntr_stuck, 0);
+	atomic_set(&hw->dcpl_oxe_cntr_dec_0, 0);
+	atomic_set(&hw->dcpl_oxe_cntr_stuck, 0);
+	atomic_set(&hw->dcpl_ixe_cntr_stuck, 0);
+	atomic_set(&hw->dcpl_success, 0);
+	atomic_set(&hw->dcpl_ibw_cntr_dec_0_62, 0);
+	atomic_set(&hw->dcpl_ibw_cntr_dec_0_62_count, 0);
+	atomic_set(&hw->dcpl_ibw_cntr_dec_0_64, 0);
+	atomic_set(&hw->dcpl_ibw_cntr_dec_0_64_count, 0);
+	atomic_set(&hw->dcpl_ibw_active_stuck, 0);
+	atomic_set(&hw->dcpl_ibw_idle_wait, 0);
+	atomic_set(&hw->dcpl_ibw_cntr_is_0, 0);
+	atomic_set(&hw->dcpl_ibw_issued, 0);
+	hw->dcpl_max_time = 0;
+	for (i = 0; i < DBINS; i++)
+		hw->dcpl_time[i] = 0;
+	hw->pri_max_md_time = 0;
+	hw->pri_max_fault_time = 0;
+	for (i = 0; i < MDBINS; i++)
+		hw->pri_md_time[i] = 0;
+	for (i = 0; i < FBINS; i++)
+		hw->pri_fault_time[i] = 0;
+
+	cass_write_ac(hw, &ac, ATU_PHYS_AC, false, false,
+		      ac.ats_vf_en, ac.ats_vf_num);
+	cass_write(hw, C_ATU_CFG_NTA, &cfg_nta, sizeof(union c_atu_cfg_nta));
+
+	hw->oxe_dummy_dma_addr = dma_map_single(&hw->cdev.pdev->dev,
+						hw->oxe_dummy_addr, PAGE_SIZE,
+						DMA_TO_DEVICE);
+	if (dma_mapping_error(&hw->cdev.pdev->dev, hw->oxe_dummy_dma_addr)) {
+		ret = -ENOMEM;
+		goto free_dummy;
+	}
+	oxe_dummy.subaddr = hw->oxe_dummy_dma_addr >> C_ADDR_SHIFT;
+	cass_write(hw, C_OXE_CFG_DUMMY_ADDR, &oxe_dummy,
+		   sizeof(union c_oxe_cfg_dummy_addr));
+
+	cass_read(hw, C_ATU_CFG_ATS, &cfg_ats.qw, sizeof(cfg_ats));
+
+	/* The simulator runs much slower so we need to change the
+	 * invalidate processing interval to a lower value.
+	 * The AMD IOMMU (in Epyc processors earlier than the 7002 series)
+	 * has bug where it returns multiple translations
+	 * with the same address if the page size is larger than the
+	 * STU size. Setting truncate_gt_stu_rsp is a workaround.
+	 * Cassini ERRATA-2431
+	 */
+	if (HW_PLATFORM_Z1(hw)) {
+		cfg_ats.inval_process_interval = 5;
+		cfg_ats.truncate_gt_stu_rsp = 1;
+	}
+
+	/* The inval_process_interval default causes IOMMU timeouts. */
+	if (HW_PLATFORM_ASIC(hw))
+		cfg_ats.inval_process_interval = inval_process_interval;
+
+	if (HW_PLATFORM_Z1(hw) || HW_PLATFORM_ASIC(hw))
+		cass_write(hw, C_ATU_CFG_ATS, &cfg_ats, sizeof(cfg_ats));
+
+	if (!cass_version(hw, CASSINI_1))
+		odp_sw_decouple = false;
+
+	if (cass_version(hw, CASSINI_1)) {
+		/* Errata 3117 */
+		union c1_atu_cfg_odp_decouple odp_decouple;
+
+		cass_read(hw, C1_ATU_CFG_ODP_DECOUPLE, &odp_decouple,
+			  sizeof(odp_decouple));
+		odp_decouple.flush_req_delay = ATU_FLUSH_REQ_DELAY;
+		odp_decouple.enable = odp_sw_decouple ? 0 : 1;
+		cass_write(hw, C1_ATU_CFG_ODP_DECOUPLE, &odp_decouple,
+			   sizeof(odp_decouple));
+	}
+
+	ret = cass_nta_cq_init(hw);
+	if (ret)
+		goto cq_init_fail;
+
+	ret = cass_nta_pri_init(hw);
+	if (ret)
+		goto pri_init_fail;
+
+	iova_cache_get();
+	cass_iommu_init(hw);
+
+	/* Register callback for C_ATU_ERR_FLG.PRB_EXPIRED bit */
+	hw->atu_err.irq = C_ATU_IRQA_MSIX_INT;
+	hw->atu_err.is_ext = false;
+	hw->atu_err.err_flags.atu.prb_expired = 1;
+	hw->atu_err.cb = cass_prb_exp_cb;
+	cxi_register_hw_errors(hw, &hw->atu_err);
+
+	return 0;
+
+pri_init_fail:
+	cass_nta_cq_fini(hw);
+
+cq_init_fail:
+	refcount_dec(&hw->refcount);
+	dma_unmap_single(&hw->cdev.pdev->dev, hw->oxe_dummy_dma_addr,
+			 PAGE_SIZE, DMA_TO_DEVICE);
+free_dummy:
+	kfree(hw->oxe_dummy_addr);
+
+	return ret;
+}
+
+static void cass_md_list_free(struct cass_ac *cac)
+{
+	struct cxi_md_priv *tmp;
+	struct cxi_md_priv *le;
+
+	mutex_lock(&cac->md_mutex);
+
+	list_for_each_entry_safe(le, tmp, &cac->md_list, md_entry) {
+		list_del(&le->md_entry);
+		refcount_dec(&cac->refcount);
+		cass_md_clear(le, false, true);
+		kfree(le);
+	}
+
+	mutex_unlock(&cac->md_mutex);
+}
+
+/**
+ * cass_cac_get - Pin a published Address Context for a reader
+ *
+ * @hw:  Cassini device
+ * @acid: Address Context ID
+ */
+struct cass_ac *cass_cac_get(struct cass_dev *hw, int acid)
+{
+	struct cass_ac *cac;
+	unsigned long flags;
+
+	if (acid < 0 || acid >= ARRAY_SIZE(hw->cac_table))
+		return NULL;
+
+	spin_lock_irqsave(&hw->cac_table_lock, flags);
+	cac = hw->cac_table[acid];
+	if (cac && !refcount_inc_not_zero(&cac->reader_refs))
+		cac = NULL;
+	spin_unlock_irqrestore(&hw->cac_table_lock, flags);
+
+	return cac;
+}
+
+void cass_cac_put(struct cass_ac *cac)
+{
+	if (cac && refcount_dec_and_test(&cac->reader_refs))
+		complete(&cac->readers_done);
+}
+
+static void cass_cac_unpublish(struct cass_dev *hw, struct cass_ac *cac)
+{
+	int acid = cac->ac.acid;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hw->cac_table_lock, flags);
+	hw->cac_table[acid] = NULL;
+	spin_unlock_irqrestore(&hw->cac_table_lock, flags);
+
+	cass_cac_put(cac);
+	wait_for_completion(&cac->readers_done);
+}
+
+static void cass_cac_free(struct cass_dev *hw, struct cass_ac *cac)
+{
+	int acid = cac->ac.acid;
+
+	if (!refcount_dec_and_test(&cac->refcount)) {
+		cxidev_WARN_ONCE(&hw->cdev, true,
+				 "Resource leaks - AC refcount not zero: %d\n",
+				 refcount_read(&cac->refcount));
+		return;
+	}
+
+	kfree(cac);
+	ida_free(&hw->atu_table, acid);
+}
+
+static void cass_ac_init(struct cass_ac *cac, const struct ac_map_opts *m_opts)
+{
+	union c_atu_cfg_ac_table *ac = &cac->cfg_ac;
+	struct cass_dev *hw = container_of(cac->lni_priv->dev,
+					   struct cass_dev, cdev);
+
+	ac->pg_table_size = cac->huge_shift - cac->page_shift;
+	ac->base_pg_size = cac->page_shift - C_ADDR_SHIFT;
+	ac->ptg_mode = cac->ptg_mode;
+	ac->do_not_cache = !!(cac->flags & CXI_MAP_NOCACHE);
+	ac->odp_en = !(cass_version(hw, CASSINI_1_0) ||
+		       (cac->flags & CXI_MAP_PIN));
+	ac->do_not_filter = hw->ac_filter_disabled;
+	ac->context_en = 1;
+
+	pr_debug("page_shift:%d huge_shift:%d base_pg_sz:%d pg_table_sz:%d\n",
+		 cac->page_shift, cac->huge_shift, ac->base_pg_size,
+		 ac->pg_table_size);
+}
+
+/**
+ * cass_lac_free() - Unmap an AC from the LNI's resource group.
+ *
+ * @lni_priv: the Logical Network Interface
+ * @lac: LAC to free
+ */
+static void cass_lac_free(struct cxi_lni_priv *lni_priv, int lac)
+{
+	struct cass_dev *hw = container_of(lni_priv->dev, struct cass_dev, cdev);
+
+	cass_set_acid(hw, lni_priv->lni.rgid, lac, C_AC_NONE);
+	cass_lac_put(hw, lni_priv->lni.rgid, lac);
+}
+
+/**
+ * cass_ac_phys_alloc - Allocate a physical Address Context for a VF
+ *
+ * This per-VF AC is analogous to ATU_PHYS_AC and is used in its place when
+ * configuring resources such as CQs that belong to a VF.
+ *
+ * @hw: CXI device
+ * @vf_en: true if allocating an AC for a VF
+ * @vf_num: VF number to allocate AC for (ignored if vf_en is false)
+ */
+int cass_ac_phys_alloc(struct cass_dev *hw, bool vf_en, int vf_num)
+{
+	int acid;
+	union c_atu_cfg_ac_table ac = {
+		.context_en = 1,
+		.ta_mode = C_ATU_PASSTHROUGH_MODE,
+		.mem_size = ~0,
+		.ats_vf_en = vf_en,
+		.ats_vf_num = vf_en ? vf_num : 0,
+	};
+
+	acid = ida_alloc_range(&hw->atu_table, 1, ATU_PHYS_AC - 1, GFP_KERNEL);
+	if (acid < 0)
+		return acid;
+
+	cass_write_ac(hw, &ac, acid, false, true, vf_en, vf_num);
+
+	return acid;
+}
+
+/**
+ * cass_ac_phys_free - Free a physical Address Context allocated by
+ * cass_ac_phys_alloc()
+ *
+ * @hw: CXI device
+ * @acid: ID of address context
+ */
+void cass_ac_phys_free(struct cass_dev *hw, int acid)
+{
+	union c_atu_cfg_ac_table ac = {};
+
+	cass_write_ac(hw, &ac, acid, false, true, false, 0);
+	ida_free(&hw->atu_table, acid);
+}
+
+/**
+ * cass_ac_alloc() - Allocate an Address Context
+ *
+ * Once the ACID is written to hardware, it cannot be freed until the LNI
+ * is freed.
+ *
+ * @lni_priv: the Logical Network Interface
+ * @m_opts: User map options
+ */
+static struct cass_ac *cass_ac_alloc(struct cxi_lni_priv *lni_priv,
+				     struct ac_map_opts *m_opts)
+{
+	int ret;
+	unsigned long irq_flags;
+	struct cass_ac *cac;
+	struct cxi_dev *dev = lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+
+	/* ac->pg_table_size is limited to 4 bits and a minimum of 8 entries */
+	if ((m_opts->huge_shift - m_opts->page_shift > MAX_PG_TABLE_SIZE) ||
+	    ((m_opts->ptg_mode != C_ATU_PTG_MODE_SGL) &&
+	     ((m_opts->huge_shift - m_opts->page_shift) < MIN_PG_TABLE_SIZE))) {
+		pr_debug("Invalid huge_shift:%d/page_shift:%d combination\n",
+			 m_opts->huge_shift, m_opts->page_shift);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* Check the associated service to see if this AC can be allocated */
+	ret = cxi_rgroup_alloc_resource(lni_priv->rgroup, CXI_RESOURCE_AC);
+	if (ret)
+		return ERR_PTR(ret);
+
+	cac = kzalloc(sizeof(*cac), GFP_KERNEL);
+	if (!cac) {
+		ret = -ENOMEM;
+		goto dec_rsrc_use;
+	}
+
+	/* AC_NONE (0) is invalid, ATU_PHYS_AC is reserved for physical
+	 * mappings
+	 */
+	ret = ida_alloc_range(&hw->atu_table, 1, ATU_PHYS_AC - 1, GFP_KERNEL);
+	if (ret < 0)
+		goto cac_free;
+
+	cac->ac.acid = ret;
+
+	ret = cass_lac_get(hw, lni_priv->lni.rgid);
+	if (ret < 0)
+		goto ac_put;
+
+	cac->ac.lac = ret;
+
+	refcount_set(&cac->refcount, 1);
+	mutex_init(&cac->ac_mutex);
+	cac->lni_priv = lni_priv;
+	cac->flags = m_opts->flags;
+	cac->ptg_mode = m_opts->ptg_mode;
+	cac->page_shift = m_opts->page_shift;
+	cac->huge_shift = m_opts->huge_shift;
+	cac->hugepage_test = m_opts->hugepage_test;
+
+	cass_ac_init(cac, m_opts);
+
+	if (m_opts->flags & CXI_MAP_ATS)
+		ret = cass_ats_init(lni_priv, m_opts, cac);
+	else
+		ret = cass_nta_init(lni_priv, m_opts, cac);
+
+	if (ret)
+		goto lac_put;
+
+	INIT_LIST_HEAD(&cac->md_list);
+	mutex_init(&cac->md_mutex);
+	refcount_set(&cac->reader_refs, 1);
+	init_completion(&cac->readers_done);
+
+	spin_lock_irqsave(&hw->cac_table_lock, irq_flags);
+	hw->cac_table[cac->ac.acid] = cac;
+	spin_unlock_irqrestore(&hw->cac_table_lock, irq_flags);
+
+	cass_write_ac(hw, &cac->cfg_ac, cac->ac.acid,
+		      m_opts->flags & CXI_MAP_USER_ADDR, false,
+		      cac->lni_priv->is_vf,
+		      cac->lni_priv->is_vf ? cac->lni_priv->vf_num : 0);
+	cass_set_acid(hw, lni_priv->lni.rgid, cac->ac.lac, cac->ac.acid);
+
+	list_add_tail(&cac->list, &lni_priv->ac_list);
+	refcount_inc(&lni_priv->refcount);
+
+	atomic_inc(&hw->stats.ac);
+
+	ac_debugfs_create(cac->ac.acid, cac, hw, lni_priv);
+
+	return cac;
+
+lac_put:
+	cass_lac_put(hw, lni_priv->lni.rgid, cac->ac.lac);
+ac_put:
+	ida_free(&hw->atu_table, cac->ac.acid);
+cac_free:
+	kfree(cac);
+dec_rsrc_use:
+	cxi_rgroup_free_resource(lni_priv->rgroup, CXI_RESOURCE_AC);
+
+	pr_debug("failed rc %d\n", ret);
+	return ERR_PTR(ret);
+}
+
+static void cass_ac_free(struct cass_dev *hw, struct cass_ac *cac)
+{
+	struct cxi_lni_priv *lni_priv = cac->lni_priv;
+
+	pr_debug("freeing ac %d\n", cac->ac.acid);
+
+	cass_cac_unpublish(hw, cac);
+
+	debugfs_remove(cac->lni_dir);
+	debugfs_remove_recursive(cac->debug_dir);
+
+	atomic_dec(&hw->stats.ac);
+	cxi_rgroup_free_resource(lni_priv->rgroup, CXI_RESOURCE_AC);
+
+	cass_md_list_free(cac);
+	cass_nta_iova_fini(cac);
+
+	if (cac->nta) {
+		cass_l1_tables_free(hw, cac->nta);
+		cass_invalidate_range(cac, 0, ATUCQ_INVALIDATE_ALL);
+		cass_nta_free(hw, cac->nta);
+	}
+
+	memset(&cac->cfg_ac, 0, sizeof(cac->cfg_ac));
+	cass_write_ac(hw, &cac->cfg_ac, cac->ac.acid, false, false,
+		      lni_priv->is_vf, lni_priv->is_vf ? lni_priv->vf_num : 0);
+
+	cass_unbind_ac(hw, cac);
+
+	cass_lac_free(lni_priv, cac->ac.lac);
+	cass_cac_free(hw, cac);
+	refcount_dec(&lni_priv->refcount);
+}
+
+/**
+ * cass_acs_disable() - Disable all the ACs of an LNI
+ *
+ * The LNI is getting released. This is the first part of the cleanup.
+ *
+ * @lni_priv: the Logical Network Interface
+ */
+void cass_acs_disable(struct cxi_lni_priv *lni_priv)
+{
+	struct cxi_dev *dev = lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cass_ac *cac;
+
+	mutex_lock(&lni_priv->ac_list_mutex);
+
+	list_for_each_entry(cac, &lni_priv->ac_list, list) {
+		cac->cfg_ac.context_en = 0;
+		cass_write_ac(hw, &cac->cfg_ac, cac->ac.acid, false, false,
+			      cac->lni_priv->is_vf,
+			      cac->lni_priv->is_vf ? cac->lni_priv->vf_num : 0);
+	}
+
+	mutex_unlock(&lni_priv->ac_list_mutex);
+
+	cass_flush_pci(hw);
+}
+
+/**
+ * cass_acs_free() - Free all ACs associated with an LNI
+ *
+ * The ac_list is used by cass_pri_fault() and cass_sync_pagetables().
+ * LNI cleanup disables all the ACs associated with the LNI and then calls
+ * inbound wait so cass_pri_fault() will not be called while cleaning up.
+ * cass_hmm_fini() will remove this process from the hmm mirrors list so
+ * cass_sync_pagetables() will not be called while cleaning up.
+ *
+ * @lni_priv: the Logical Network Interface
+ */
+void cass_acs_free(struct cxi_lni_priv *lni_priv)
+{
+	struct cass_ac *cac;
+	struct cxi_dev *dev = lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+
+	mutex_lock(&lni_priv->ac_list_mutex);
+
+	while ((cac = list_first_entry_or_null(&lni_priv->ac_list,
+					       struct cass_ac, list))) {
+		list_del(&cac->list);
+		mutex_unlock(&lni_priv->ac_list_mutex);
+		cass_ac_free(hw, cac);
+		mutex_lock(&lni_priv->ac_list_mutex);
+	}
+
+	mutex_unlock(&lni_priv->ac_list_mutex);
+
+	if (dev->is_physfn)
+		cass_flush_pci(hw);
+}
+
+/**
+ * cass_iova_alloc() - Allocate an IO virtual address in an Address Context
+ *
+ * Find an AC with matching characteristics and allocate an IOVA.
+ *
+ * @lni_priv: the Logical Network Interface
+ * @m_opts: User map options
+ */
+static struct cass_ac *cass_iova_alloc(struct cxi_lni_priv *lni_priv,
+				       struct ac_map_opts *m_opts)
+{
+	u64 iova;
+	struct list_head *ptr;
+	u32 masked_flags = m_opts->flags & ATU_FLAGS_MASK;
+
+	list_for_each(ptr, &lni_priv->ac_list) {
+		struct cass_ac *cac = list_entry(ptr, struct cass_ac, list);
+		u32 cac_msk_flags = cac->flags & ATU_FLAGS_MASK;
+
+		if ((m_opts->page_shift == cac->page_shift) &&
+		    (m_opts->huge_shift == cac->huge_shift) &&
+				(m_opts->ptg_mode == cac->ptg_mode) &&
+				(masked_flags == cac_msk_flags) &&
+				!m_opts->hugepage_test) {
+			if (m_opts->flags & CXI_MAP_ATS) {
+				iova = m_opts->va_start;
+			} else {
+				iova = cass_nta_iova_alloc(cac, m_opts);
+				if (!iova)
+					continue;
+			}
+
+			m_opts->iova = iova;
+
+			return cac;
+		}
+	}
+
+	return ERR_PTR(-ENOSPC);
+}
+
+/* Find or allocate an AC matching the options */
+static struct cass_ac *get_matching_ac(struct cxi_lni_priv *lni_priv,
+				       struct ac_map_opts *m_opts)
+{
+	struct cass_ac *cac;
+
+	mutex_lock(&lni_priv->ac_list_mutex);
+
+	cac = cass_iova_alloc(lni_priv, m_opts);
+	if (IS_ERR(cac))
+		cac = cass_ac_alloc(lni_priv, m_opts);
+
+	mutex_unlock(&lni_priv->ac_list_mutex);
+
+	return cac;
+}
+
+static void cass_init_md(const struct ac_map_opts *m_opts,
+			 struct cxi_md_priv *md_priv)
+{
+	md_priv->md.iova = m_opts->iova;
+	md_priv->md.va = m_opts->va_start;
+	md_priv->md.len = m_opts->va_len;
+	md_priv->md.page_shift = m_opts->page_shift;
+	md_priv->md.huge_shift = m_opts->huge_shift;
+	md_priv->md.lac = md_priv->cac->ac.lac;
+	md_priv->olen = m_opts->va_len;
+	md_priv->flags = m_opts->flags & ~(CXI_MAP_FAULT | CXI_MAP_PREFETCH);
+
+	/* Since we don't mirror the page tables when CXI_MAP_ALLOC_MD
+	 * is set, the first call to cxi_update_iov needs to lock.
+	 */
+	if (m_opts->flags & CXI_MAP_ALLOC_MD)
+		md_priv->need_lock = true;
+}
+
+/* Add an MD to an LNI */
+static void cass_add_md(struct cxi_lni_priv *lni_priv,
+			const struct ac_map_opts *m_opts,
+			struct cxi_md_priv *md_priv)
+{
+	struct cass_ac *cac = md_priv->cac;
+	struct cass_dev *hw = container_of(cac->lni_priv->dev, struct cass_dev, cdev);
+
+	mutex_lock(&cac->md_mutex);
+	list_add_tail(&md_priv->md_entry, &cac->md_list);
+	refcount_inc(&cac->refcount);
+	atomic_inc(&hw->stats.md);
+	mutex_unlock(&cac->md_mutex);
+}
+
+/**
+ * cass_remove_md() - Remove an MD from its LNI, clear and invalidate
+ *                    page table entries.
+ *
+ * @md_priv: Private memory descriptor
+ */
+static void cass_remove_md(struct cxi_md_priv *md_priv)
+{
+	struct cass_ac *cac = md_priv->cac;
+	struct cass_dev *hw = container_of(cac->lni_priv->dev, struct cass_dev, cdev);
+
+	mutex_lock(&md_priv->cac->md_mutex);
+	list_del(&md_priv->md_entry);
+	refcount_dec(&cac->refcount);
+	atomic_dec(&hw->stats.md);
+	mutex_unlock(&md_priv->cac->md_mutex);
+
+	cass_md_clear(md_priv, true, true);
+}
+
+static void cass_md_iova_free(struct cxi_md_priv *md_priv)
+{
+	struct cass_ac *cac = md_priv->cac;
+
+	if (cac->iovad)
+		FREE_IOVA_FAST(cac->iovad, md_priv->md.iova, md_priv->olen);
+}
+
+static int cass_bvec(struct cass_dev *hw, const struct iov_iter *iter,
+		     size_t *len, u64 *va)
+{
+	int i;
+	int last_seg = iter->nr_segs - 1;
+	const struct bio_vec *bvec = iter->bvec;
+
+	*va = (u64)page_address(bvec->bv_page);
+
+	for (i = 0; i < iter->nr_segs; i++) {
+		if (i && (bvec->bv_offset & ~PAGE_MASK)) {
+			cxidev_err(&hw->cdev, "bvec[%d] offset not aligned %d\n", i,
+				   bvec->bv_offset);
+			return -EINVAL;
+		}
+
+		if (i < last_seg && ((bvec->bv_offset + bvec->bv_len) &
+						~PAGE_MASK)) {
+			cxidev_err(&hw->cdev, "bvec[%d] contains a hole\n", i);
+			return -EINVAL;
+		}
+		bvec++;
+	}
+
+	*len = iter->nr_segs * PAGE_SIZE;
+
+	return 0;
+}
+
+static int cass_kvec(struct cass_dev *hw, const struct iov_iter *iter,
+		     size_t *len, u64 *va)
+{
+	int i;
+	int alen;
+	int tlen = 0;
+	int last_seg = iter->nr_segs - 1;
+	const struct kvec *kvec = iter->kvec;
+
+	*va = (u64)kvec->iov_base & PAGE_MASK;
+
+	for (i = 0; i < iter->nr_segs; i++) {
+		/*
+		 * Only the first kvec may be unaligned. The first and last
+		 * kvec's lengths can be less that a page but the rest need
+		 * to be full pages. The first base + len must be aligned.
+		 */
+		if (i && ((u64)kvec->iov_base & ~PAGE_MASK)) {
+			cxidev_err(&hw->cdev, "kvec[%d] base not aligned %p\n", i,
+				   kvec->iov_base);
+			return -EINVAL;
+		}
+
+		if (i < last_seg && (((u64)kvec->iov_base + kvec->iov_len) &
+					~PAGE_MASK)) {
+			cxidev_err(&hw->cdev, "kvec[%d] contains a hole\n", i);
+			return -EINVAL;
+		}
+
+		alen = PAGE_ALIGN(kvec->iov_len);
+		tlen += alen;
+		kvec++;
+	}
+
+	*len = tlen;
+
+	return 0;
+}
+
+/* cass_sgtable_is_valid() - Check for invalid sgt
+ *
+ * Requirements for entries in sgt:
+ * one entry:
+ *     offset < PAGE_SIZE
+ * multiple entries:
+ *     first:  offset < PAGE_SIZE, (len + offset) % PAGE_SIZE
+ *     middle: offset == 0, len % PAGE_SIZE
+ *     last:   offset == 0
+ */
+static bool cass_sgtable_is_valid(struct cass_dev *hw,
+				  const struct sg_table *sgt, size_t *length)
+{
+	int i;
+	struct scatterlist *sg;
+	int last_entry = sgt->nents - 1;
+
+	*length = 0;
+
+	for_each_sgtable_dma_sg(sgt, sg, i) {
+		size_t len = sg_dma_len(sg);
+
+		if (sg->offset > PAGE_SIZE) {
+			cxidev_err(&hw->cdev,
+				   "Entry %d offset:%x > PAGE_SIZE\n",
+				   i, sg->offset);
+
+			return false;
+		}
+
+		/* first of multiple entries */
+		if ((!i && i != last_entry && ((len + sg->offset) % PAGE_SIZE)) ||
+		    /* last entry */
+		    (i && i == last_entry && sg->offset) ||
+		    /* middle entries */
+		    (i && i < last_entry && (sg->offset ||
+					     len % PAGE_SIZE))) {
+			cxidev_err(&hw->cdev, "Hole in sg_table at entry %d offset:%x len:%lx\n",
+				   i, sg->offset, len);
+
+			return false;
+		}
+
+		*length += ALIGN(len + sg->offset, PAGE_SIZE);
+	}
+
+	return true;
+}
+
+/**
+ * cass_mirror_range() - Mirror an address range
+ *
+ * @md_priv: Private memory descriptor
+ * @m_opts:  User options
+ *
+ * @return: 0 on success or -ENOMEM
+ */
+int cass_mirror_range(struct cxi_md_priv *md_priv, struct ac_map_opts *m_opts)
+{
+	if (!(m_opts->flags & CXI_MAP_USER_ADDR))
+		return cass_nta_mirror_kern(md_priv, NULL, true);
+
+	if (m_opts->flags & CXI_MAP_ATS)
+		return cass_ats_md_init(md_priv, m_opts);
+
+	if (m_opts->flags & CXI_MAP_DEVICE)
+		return cass_mirror_device(md_priv, m_opts);
+
+	if (m_opts->hugepage_test)
+		return cass_mirror_hp(m_opts, md_priv);
+
+	if (m_opts->flags & CXI_MAP_PIN)
+		return cass_pin_mirror(md_priv, m_opts);
+
+	return cass_mmu_notifier_insert(md_priv, m_opts);
+}
+
+static void cass_unpin(struct cxi_md_priv *md_priv)
+{
+	int npages = md_priv->md.len >> PAGE_SHIFT;
+
+	if (!(md_priv->flags & CXI_MAP_PIN))
+		return;
+
+	if (!md_priv->pages)
+		return;
+
+	unpin_user_pages(md_priv->pages, npages);
+	kvfree(md_priv->pages);
+	md_priv->pages = NULL;
+}
+
+/**
+ * cass_md_clear() - Clear a memory descriptor
+ *
+ * Remove page table entries for NTA mode
+ * Unpin ATS or device pinned pages
+ *
+ * @md_priv: Private memory descriptor
+ * @inval: Invalidate this iova range
+ * @need_lock: Indicates whether the ac_mutex is required
+ */
+void cass_md_clear(struct cxi_md_priv *md_priv, bool inval, bool need_lock)
+{
+	struct cass_dev *hw = container_of(md_priv->cac->lni_priv->dev,
+					   struct cass_dev, cdev);
+
+	cass_notifier_cleanup(md_priv);
+
+	if (md_priv->flags & CXI_MAP_ATS)
+		return cass_unpin(md_priv);
+
+	/* If inval is false we have been called from cass_md_list_free()
+	 * which is only called when the AC is cleaning up so we don't
+	 * need to invalidate here as we will invalidate the whole AC range.
+	 */
+	cass_cond_lock(&md_priv->cac->ac_mutex, need_lock);
+	cass_clear_range(md_priv, md_priv->md.iova,
+			 md_priv->md.len);
+	cass_cond_unlock(&md_priv->cac->ac_mutex, need_lock);
+
+	if (inval) {
+		cass_invalidate_range(md_priv->cac, md_priv->md.iova,
+				      md_priv->md.len);
+		atomic_inc(&hw->dcpl_md_clear_inval);
+	}
+
+	if (md_priv->flags & CXI_MAP_DEVICE)
+		cass_device_put_pages(md_priv);
+	else
+		cass_unpin(md_priv);
+}
+
+static int largest_hp_size(struct vm_area_struct *vma, u64 end)
+{
+	int vma_hs;
+	int hs = INT_MAX;
+	ulong vm_end_prev;
+#ifdef VMA_ITERATOR
+	VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_start);
+#endif
+
+	while (true) {
+		if (!is_vm_hugetlb_page(vma))
+			return PMD_SHIFT;
+
+		vma_hs = huge_page_shift(hstate_vma(vma));
+
+		hs = vma_hs < hs ? vma_hs : hs;
+
+		if (vma->vm_end >= end)
+			return hs;
+
+		vm_end_prev = vma->vm_end;
+#ifdef VMA_ITERATOR
+		vma = vma_next(&vmi);
+#else
+		vma = vma->vm_next;
+#endif
+		/* full range is not backed by vmas */
+		if (!vma)
+			return PMD_SHIFT;
+
+		/* found a hole */
+		if (vma->vm_start > vm_end_prev)
+			return PMD_SHIFT;
+
+		/* not in range */
+		if (vma->vm_start >= end)
+			break;
+	}
+
+	return hs;
+}
+
+static int align_and_page_shift(struct ac_map_opts *m_opts)
+{
+	/* To support hugepage sizes larger than 128MB, we need
+	 * a larger base page size.
+	 */
+	if ((m_opts->huge_shift - m_opts->page_shift) >
+			(MAX_PG_TABLE_SIZE - hugepage_threshold))
+		m_opts->page_shift = m_opts->huge_shift -
+				MAX_PG_TABLE_SIZE + hugepage_threshold;
+
+	/* A hugepage is backing this address so align
+	 * to and map the full hugepage.
+	 */
+	return m_opts->huge_shift;
+}
+
+int cass_cpu_page_size(struct cass_dev *hw, struct ac_map_opts *m_opts,
+		       struct mm_struct *mm, uintptr_t va,
+		       const struct cxi_md_hints *hints, int *align_shift)
+{
+	int ret = 0;
+	struct vm_area_struct *vma;
+
+	/* ODP hint from user for the hugepage size */
+	if (hints && !hints->page_shift && hints->huge_shift &&
+	    !(m_opts->flags & (CXI_MAP_PIN | CXI_MAP_FAULT | CXI_MAP_PREFETCH))) {
+		m_opts->huge_shift = hints->huge_shift;
+		*align_shift = align_and_page_shift(m_opts);
+
+		return 0;
+	}
+
+	mmap_read_lock(mm);
+	vma = find_vma(mm, va);
+	if (!vma) {
+		cxidev_warn(&hw->cdev, "No VMA covering 0x%016lx)\n", va);
+		ret = -EFAULT;
+		goto unlock;
+	}
+
+	/* Test arbitrary page sizes supplied by the user. The contiguous
+	 * pages will be mmapped from a 1G hugepage and combined into larger
+	 * base pages and smaller hugepages.
+	 */
+	if (hints && hints->page_shift && is_vm_hugetlb_page(vma)) {
+		m_opts->huge_shift = huge_page_shift(hstate_vma(vma));
+
+		pr_debug("Testing huge shift:%d bs:%d hs:%d\n",
+			 m_opts->huge_shift, hints->page_shift,
+			 hints->huge_shift);
+
+		if (hints->huge_shift > m_opts->huge_shift) {
+			cxidev_warn(&hw->cdev, "Hints huge_shift (%d) > vma's huge_shift (%d)\n",
+				    hints->huge_shift, m_opts->huge_shift);
+			ret = -EINVAL;
+			goto unlock;
+		}
+
+		m_opts->page_shift = hints->page_shift;
+		m_opts->huge_shift = hints->huge_shift;
+		*align_shift = hints->huge_shift;
+		m_opts->hugepage_test = true;
+		goto unlock;
+	}
+
+	if (is_vm_hugetlb_page(vma)) {
+		m_opts->huge_shift = largest_hp_size(vma, m_opts->va_end);
+		m_opts->is_huge_page = true;
+		*align_shift = align_and_page_shift(m_opts);
+	} else {
+		int vms_fs = ffsl(vma->vm_start);
+		int vml_fs = ffsl(vma->vm_end - vma->vm_start);
+
+		/* Small regions may not be aligned to the default alignment
+		 * passed in so limit alignment to the VMA.
+		 */
+		if (vms_fs < *align_shift || vml_fs < *align_shift)
+			*align_shift = min_t(int, vms_fs, vml_fs);
+	}
+
+unlock:
+	mmap_read_unlock(mm);
+
+	return ret;
+}
+
+/**
+ * cass_align_start_len() - Align the va and len to align_shift page size
+ *
+ * @m_opts: User map options
+ * @va: Virtual address
+ * @len: Length of address
+ * @align_shift: Power of two to align to
+ */
+void cass_align_start_len(struct ac_map_opts *m_opts, uintptr_t va, size_t len,
+			  int align_shift)
+{
+	size_t page_size = BIT(align_shift);
+
+	m_opts->va_start = round_down(va, page_size);
+	m_opts->va_len = round_up(va + len, page_size) - m_opts->va_start;
+	m_opts->va_end = m_opts->va_start + m_opts->va_len;
+}
+
+/**
+ * cxi_phys_lac_alloc_vf() - Allocate a physical LAC for the given LNI for a VF.
+ *
+ * @lni: the Logical Network Interface
+ * @return: LAC or negative errno
+ */
+static int cxi_phys_lac_alloc_vf(struct cxi_lni *lni)
+{
+	struct cxi_lni_priv_vf *lni_priv_vf = container_of(lni, struct cxi_lni_priv_vf, lni);
+
+	return lni_priv_vf->phys_lac;
+}
+
+/**
+ * cxi_phys_lac_alloc() - Allocate a physical LAC for the given LNI.
+ *                        Exported to allow clients to perform DMA using physical addresses.
+ *
+ * @lni: the Logical Network Interface
+ * @return: LAC or negative errno
+ */
+int cxi_phys_lac_alloc(struct cxi_lni *lni)
+{
+	struct cxi_lni_priv *lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(lni_priv->dev, struct cass_dev, cdev);
+	int lac;
+
+	if (!cdev->is_physfn)
+		return cxi_phys_lac_alloc_vf(lni);
+
+	lac = cass_lac_get(hw, lni_priv->lni.rgid);
+	if (lac < 0)
+		return lac;
+
+	/* Program the phys_ac that was already selected at the lni allocation */
+	cass_set_acid(hw, lni_priv->lni.rgid, lac, lni_priv->phys_ac);
+
+	return lac;
+}
+EXPORT_SYMBOL(cxi_phys_lac_alloc);
+
+/**
+ * cxi_phys_lac_free() - Unmap a physical LAC
+ *
+ * @lni: the Logical Network Interface
+ * @lac: LAC to free
+ */
+void cxi_phys_lac_free(struct cxi_lni *lni, int lac)
+{
+	struct cxi_lni_priv *lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *cdev = lni_priv->dev;
+
+	/* VFs do not free physical LACs. They will be freed when the LNI is freed */
+	if (!cdev->is_physfn)
+		return;
+
+	cass_lac_free(lni_priv, lac);
+}
+EXPORT_SYMBOL(cxi_phys_lac_free);
+
+/**
+ * cxi_clear_md() - Zero page table entries and invalidate a memory descriptor
+ *
+ * @md: Memory Descriptor used in network operations
+ */
+int cxi_clear_md(struct cxi_md *md)
+{
+	struct cxi_md_priv *md_priv;
+
+	if (!md)
+		return -EINVAL;
+
+	md_priv = container_of(md, struct cxi_md_priv, md);
+
+	pr_debug("va:%llx iova:%llx len:%lx\n", md->va, md->iova, md->len);
+
+	cass_md_clear(md_priv, true, false);
+	cass_dma_unmap_pages(md_priv);
+	md_priv->external_sgt_owner = false;
+	md_priv->sgt = NULL;
+
+	return 0;
+}
+EXPORT_SYMBOL(cxi_clear_md);
+
+/**
+ * cxi_update_iov() - Map a list of virtual address or pages in an IO
+ *                    virtual address space into an existing memory
+ *                    descriptor.
+ *
+ * @md: Memory descriptor
+ * @iter: List of virtual addresses to map - supports ITER_KVEC and ITER_BVEC
+ */
+int cxi_update_iov(struct cxi_md *md, const struct iov_iter *iter)
+{
+	u64 va;
+	int ret;
+	size_t len;
+	struct cxi_md_priv *md_priv;
+	struct cxi_lni_priv *lni_priv;
+	struct cxi_dev *dev;
+	struct cass_dev *hw;
+
+	if (!md)
+		return -EINVAL;
+
+	md_priv = container_of(md, struct cxi_md_priv, md);
+	lni_priv = md_priv->lni_priv;
+	dev = lni_priv->dev;
+	hw = container_of(dev, struct cass_dev, cdev);
+
+	if (iov_iter_is_bvec(iter))
+		ret = cass_bvec(hw, iter, &len, &va);
+	else
+		ret = cass_kvec(hw, iter, &len, &va);
+	if (ret)
+		return ret;
+
+	if (len > md_priv->olen)
+		return -EINVAL;
+
+	md->len = len;
+
+	ret = cass_nta_mirror_kern(md_priv, iter, md_priv->need_lock);
+
+	pr_debug("iovs:%ld len:0x%lx iova:%llx\n", iter->nr_segs, md->len,
+		 md->iova);
+
+	return ret;
+
+}
+EXPORT_SYMBOL(cxi_update_iov);
+
+/**
+ * cxi_map_iov() - Map a list of virtual address or pages in an IO
+ *                 virtual address space
+ *
+ * @lni: the Logical Network Interface
+ * @iter: List of virtual addresses to map - supports ITER_KVEC and ITER_BVEC
+ * @flags: various options affecting the map
+ */
+struct cxi_md *cxi_map_iov(struct cxi_lni *lni, const struct iov_iter *iter,
+			   u32 flags)
+{
+	u64 va;
+	int ret;
+	struct cxi_lni_priv *lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	struct cxi_md_priv *md_priv;
+	struct cxi_md *md;
+	struct ac_map_opts m_opts = {
+		.page_shift = PAGE_SHIFT,
+		.huge_shift = PMD_SHIFT
+	};
+	struct cass_ac *cac;
+
+	/* currently unsupported flags */
+	if (flags & CXI_MAP_ATS)
+		return ERR_PTR(-EINVAL);
+
+	if (iov_iter_is_bvec(iter))
+		ret = cass_bvec(hw, iter, &m_opts.va_len, &va);
+	else
+		ret = cass_kvec(hw, iter, &m_opts.va_len, &va);
+	if (ret)
+		return ERR_PTR(ret);
+
+	m_opts.va_start = 0;
+	m_opts.flags = flags;
+	m_opts.va_end = m_opts.va_start + m_opts.va_len;
+	m_opts.ptg_mode = default_ptg_mode;
+
+	cac = get_matching_ac(lni_priv, &m_opts);
+	if (IS_ERR(cac))
+		return ERR_PTR(PTR_ERR(cac));
+
+	md_priv = kzalloc(sizeof(*md_priv), GFP_KERNEL);
+	if (md_priv == NULL)
+		return ERR_PTR(-ENOMEM);
+	refcount_set(&md_priv->refcount, 1);
+	md = &md_priv->md;
+	md_priv->device = &hw->cdev.pdev->dev;
+
+	/* Get an MD ID, above CXI_MD_NONE */
+	ret = ida_alloc_min(&hw->md_index_table, 1, GFP_KERNEL);
+	if (ret < 0) {
+		cxidev_err(cdev, "ida_alloc_min failed %d\n", ret);
+		goto md_free;
+	}
+	md->id = ret;
+
+	md_priv->cac = cac;
+	md_priv->lni_priv = lni_priv;
+
+	cass_init_md(&m_opts, md_priv);
+
+	ret = cass_nta_mirror_kern(md_priv, iter, true);
+	if (ret)
+		goto mirror_range_error;
+
+	cass_add_md(lni_priv, &m_opts, md_priv);
+
+	pr_debug("iovs:%ld len:0x%lx iova:%llx\n", iter->nr_segs, md->len,
+		 md->iova);
+
+	refcount_inc(&lni_priv->refcount);
+
+	return md;
+
+mirror_range_error:
+	cass_md_iova_free(md_priv);
+	ida_free(&hw->md_index_table, md->id);
+md_free:
+	kfree(md_priv);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(cxi_map_iov);
+
+/**
+ * cxi_map_sgtable_vf() - Map an sg table to an IO virtual address space for a VF
+ *
+ * @lni: The Logical Network Interface
+ * @sgt: The sg_table object. It is expected to be dma mapped.
+ * @flags: Various options affecting the map
+ *
+ * @return: memory descriptor or error code
+ */
+static struct cxi_md *cxi_map_sgtable_vf(struct cxi_lni *lni,
+					 struct sg_table *sgt, u32 flags)
+{
+	struct cxi_lni_priv *lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_lni_priv_vf *lni_priv_vf = container_of(lni, struct cxi_lni_priv_vf, lni);
+	struct cxi_dev *dev = lni_priv_vf->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cxi_md_priv_vf *md_priv_vf;
+	struct cxi_atu_map_sgt_cmd *cmd;
+	struct cxi_atu_map_sgt_resp resp = {};
+	size_t resp_len = sizeof(resp);
+	struct scatterlist *sg;
+	int i;
+	int rc;
+	size_t sgt_len;
+
+	if (!sgt)
+		return ERR_PTR(-EINVAL);
+
+	if (!cass_sgtable_is_valid(hw, sgt, &sgt_len))
+		return ERR_PTR(-EINVAL);
+
+	sg = sgt->sgl;
+
+	md_priv_vf = kzalloc(sizeof(*md_priv_vf), GFP_KERNEL);
+	if (!md_priv_vf)
+		return ERR_PTR(-ENOMEM);
+
+	md_priv_vf->lni_priv = lni_priv;
+	md_priv_vf->device = &hw->cdev.pdev->dev;
+	md_priv_vf->md.va = 0;
+	md_priv_vf->md.len = sgt_len;
+	md_priv_vf->md.page_shift = PAGE_SHIFT;
+	md_priv_vf->md.huge_shift = PMD_SHIFT;
+	md_priv_vf->flags = flags & ~(CXI_MAP_FAULT | CXI_MAP_PREFETCH);
+	md_priv_vf->sgt = sgt;
+	md_priv_vf->pages = NULL;
+	md_priv_vf->npages = 0;
+
+	/* Send sgt to the PF to run cxi_map_sgtable there */
+	if (sgt->nents > CXI_ATU_SGT_MAX_ENTRIES) {
+		rc = -E2BIG;
+		cxidev_err(&hw->cdev, "SGT has too many entries: %u\n", sgt->nents);
+		goto free_md_priv;
+	}
+
+	pr_debug("VF map: md:%p lni:%u rgid:%u nents:%u len:0x%lx flags:0x%x\n",
+		 md_priv_vf, lni_priv_vf->lni.id, lni_priv_vf->lni.rgid, sgt->nents, sgt_len,
+		 flags);
+
+	cmd = kvzalloc(sizeof(*cmd), GFP_KERNEL);
+	if (!cmd) {
+		rc = -ENOMEM;
+		goto free_md_priv;
+	}
+
+	cmd->op = CXI_OP_ATU_MAP_SGT;
+	cmd->lni = lni_priv_vf->lni.id;
+	cmd->flags = flags;
+	cmd->nents = sgt->nents;
+
+	pr_debug("Sending SGT with %d entries to PF\n", sgt->nents);
+	for (i = 0; i < sgt->nents; i++) {
+		cmd->sge[i].dma_addr = sg_dma_address(sg);
+		cmd->sge[i].dma_len = sg_dma_len(sg);
+		cmd->sge[i].offset = sg->offset;
+
+		sg = sg_next(sg);
+	}
+
+	rc = cxi_send_msg_to_pf(dev, cmd, sizeof(*cmd), &resp, &resp_len);
+	kvfree(cmd);
+	if (rc)
+		goto free_md_priv;
+
+	/* Set the iova and the lac retrieved from the PF */
+	md_priv_vf->md.iova = resp.md.iova;
+	md_priv_vf->md.lac = resp.md.lac;
+	md_priv_vf->md.id = resp.md.id;
+	pr_debug("VF map resp: md:%d iova:%llx lac:%u resp_len:%zu\n",
+		 md_priv_vf->md.id, md_priv_vf->md.iova,
+		 md_priv_vf->md.lac, resp_len);
+
+	return &md_priv_vf->md;
+
+free_md_priv:
+	kfree(md_priv_vf);
+	return ERR_PTR(rc);
+}
+
+static int cxi_unmap_sgtable_vf(struct cxi_dev *dev,
+				struct cxi_md_priv_vf *md_priv_vf,
+				struct cass_dev *hw, int md_id)
+{
+	struct cxi_atu_unmap_cmd cmd = {
+		.op = CXI_OP_ATU_UNMAP,
+		.id = md_priv_vf->md.id,
+	};
+	size_t resp_len = 0;
+	int rc;
+
+	rc = cxi_send_msg_to_pf(dev, &cmd, sizeof(cmd), NULL, &resp_len);
+	if (rc)
+		cxidev_err(&hw->cdev, "VF ATU_UNMAP failed md:%d pf_md:%d rc:%d\n",
+			   md_id, md_priv_vf->md.id, rc);
+
+	return rc;
+}
+
+/**
+ * cxi_map_sgtable() - Map an sg table to an IO virtual address space
+ *
+ * @lni: The Logical Network Interface
+ * @sgt: The sg_table object. It is expected to be dma mapped.
+ * @flags: Various options affecting the map
+ *
+ * @return: memory descriptor or error code
+ */
+struct cxi_md *cxi_map_sgtable(struct cxi_lni *lni, struct sg_table *sgt,
+			       u32 flags)
+{
+	int ret;
+	size_t len;
+	struct cxi_md *md;
+	struct cass_ac *cac;
+	struct cxi_md_priv *md_priv;
+	struct cxi_lni_priv *lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	struct ac_map_opts m_opts = {
+		.page_shift = PAGE_SHIFT,
+		.huge_shift = PMD_SHIFT
+	};
+
+	/* unsupported flags */
+	if (flags & CXI_MAP_ATS)
+		return ERR_PTR(-EINVAL);
+
+	if (!cass_sgtable_is_valid(hw, sgt, &len))
+		return ERR_PTR(-EINVAL);
+
+	if (!cdev->is_physfn)
+		return cxi_map_sgtable_vf(lni, sgt, flags);
+
+	m_opts.va_start = 0;
+	m_opts.flags = flags;
+	m_opts.va_len = len;
+	m_opts.va_end = m_opts.va_len;
+	m_opts.ptg_mode = default_ptg_mode;
+
+	cac = get_matching_ac(lni_priv, &m_opts);
+	if (IS_ERR(cac))
+		return ERR_PTR(PTR_ERR(cac));
+
+	md_priv = kzalloc(sizeof(*md_priv), GFP_KERNEL);
+	if (md_priv == NULL)
+		return ERR_PTR(-ENOMEM);
+	refcount_set(&md_priv->refcount, 1);
+	md = &md_priv->md;
+	md_priv->device = &hw->cdev.pdev->dev;
+
+	/* Get an MD ID, above CXI_MD_NONE */
+	ret = ida_alloc_min(&hw->md_index_table, 1, GFP_KERNEL);
+	if (ret < 0) {
+		cxidev_err(cdev, "ida_alloc_min failed %d\n", ret);
+		goto md_free;
+	}
+	md->id = ret;
+
+	md_priv->sgt = sgt;
+	md_priv->external_sgt_owner = true;
+	md_priv->cac = cac;
+	md_priv->lni_priv = lni_priv;
+
+	cass_init_md(&m_opts, md_priv);
+
+	pr_debug("va:%llx va_len:%lx iova:%0llx iova_base:%llx md:%d ac:%d flags:%x\n",
+		 m_opts.va_start, m_opts.va_len, m_opts.iova, cac->iova_base,
+		 md->id, md_priv->cac->ac.acid, flags);
+
+	ret = cass_nta_mirror_sgt(md_priv, true);
+	if (ret)
+		goto mirror_range_error;
+
+	cass_add_md(lni_priv, &m_opts, md_priv);
+
+	pr_debug("nents:%u len:0x%lx iova:%llx\n", sgt->nents, md->len,
+		 md->iova);
+
+	refcount_inc(&lni_priv->refcount);
+
+	return md;
+
+mirror_range_error:
+	cass_md_iova_free(md_priv);
+	ida_free(&hw->md_index_table, md->id);
+md_free:
+	kfree(md_priv);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(cxi_map_sgtable);
+
+/**
+ * cxi_update_sgtable() - Update the specified address range contained in MD
+ *
+ * @md: Memory Descriptor used in network operations
+ * @sgt: The sg_table object. It is expected to be dma mapped.
+ *
+ * @return: 0 on success or negative error value
+ */
+int cxi_update_sgtable(struct cxi_md *md, struct sg_table *sgt)
+{
+	int ret;
+	size_t len;
+	struct cxi_md_priv *md_priv = container_of(md, struct cxi_md_priv, md);
+	struct cxi_dev *cdev = md_priv->lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+
+	if (!cass_sgtable_is_valid(hw, sgt, &len))
+		return -EINVAL;
+
+	if (len > md_priv->olen) {
+		pr_debug("Address range not bounded by MD\n");
+		return -EINVAL;
+	}
+
+	md_priv->sgt = sgt;
+	md_priv->external_sgt_owner = true;
+
+	pr_debug("nents:%u len:0x%lx iova:%llx\n", sgt->nents, md->len,
+		 md->iova);
+
+	ret = cass_nta_mirror_sgt(md_priv, md_priv->need_lock);
+	if (ret)
+		return ret;
+
+	md_priv->need_lock = false;
+
+	return 0;
+}
+EXPORT_SYMBOL(cxi_update_sgtable);
+
+/**
+ * cass_virt_to_pages() - Get page structures for kernel virtual addresses
+ *
+ * @hw: Cassini device for error reporting
+ * @va: Starting virtual address (will be page-aligned)
+ * @npages: Number of pages to get
+ * @pages: Array to store page pointers
+ *
+ * @return: 0 on success, negative error code on failure
+ */
+static int cass_virt_to_pages(struct cass_dev *hw, uintptr_t va, int npages,
+			      struct page **pages)
+{
+	int i;
+	uintptr_t addr = va & PAGE_MASK;
+
+	for (i = 0; i < npages; i++, addr += PAGE_SIZE) {
+		if (is_vmalloc_addr((void *)addr))
+			pages[i] = vmalloc_to_page((void *)addr);
+		else
+			pages[i] = virt_to_page(addr);
+
+		if (!pages[i]) {
+			cxidev_err(&hw->cdev, "Failed to get page for addr 0x%lx\n", addr);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static struct cxi_md *cxi_map_vf(struct cxi_lni *lni, uintptr_t va, size_t len,
+				 u32 flags, const struct cxi_md_hints *hints)
+{
+	struct cxi_lni_priv_vf *lni_priv_vf = container_of(lni, struct cxi_lni_priv_vf, lni);
+	struct cxi_dev *dev = lni_priv_vf->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cxi_md *md;
+	struct cxi_md_priv_vf *md_priv_vf;
+	/* For now, we do not support hugepages. This will come later when we add the ATS
+	 * for both VF and PF: NETCASSINI-7859.
+	 * In this effort we will reuse code to align pages and to support huge pages too
+	 */
+	const int npages = (PAGE_ALIGN(va + len) - (va & PAGE_MASK)) >> PAGE_SHIFT;
+	struct page **pages;
+	struct sg_table *sgt;
+	int rc;
+
+	if (flags & CXI_MAP_USER_ADDR && !(flags & CXI_MAP_PIN)) {
+		cxidev_err(&hw->cdev, "VF does not support ODP\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (flags & CXI_MAP_DEVICE) {
+		cxidev_err(&hw->cdev, "TODO: VF does not support device memory\n");
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	if (!va) {
+		cxidev_err(&hw->cdev, "NULL virtual address\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!pages) {
+		rc = -ENOMEM;
+		return ERR_PTR(rc);
+	}
+
+	/* Get the pages and pin them if requested */
+	if (flags & CXI_MAP_PIN) {
+		rc = pin_user_pages_fast(va & PAGE_MASK, npages, FOLL_WRITE | FOLL_LONGTERM, pages);
+		if (rc != npages) {
+			cxidev_err(&hw->cdev, "pin_user_pages_fast failed: %d/%d\n", rc, npages);
+			if (rc > 0)
+				unpin_user_pages(pages, rc);
+			rc = rc < 0 ? rc : -EFAULT;
+			goto free_pages;
+		}
+	} else {
+		rc = cass_virt_to_pages(hw, va, npages, pages);
+		if (rc)
+			goto free_pages;
+	}
+
+	/* Now we DMA-map the pages using a scatter-gather table */
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		rc = -ENOMEM;
+		goto unpin_pages;
+	}
+
+	rc = sg_alloc_table_from_pages_segment(sgt, pages, npages,
+					       va & ~PAGE_MASK, len,
+					       UINT_MAX, GFP_KERNEL);
+	if (rc)
+		goto free_sgt;
+
+	rc = dma_map_sgtable(&hw->cdev.pdev->dev, sgt, DMA_BIDIRECTIONAL, 0);
+	if (rc) {
+		cxidev_err(&hw->cdev, "dma_map_sgtable failed\n");
+		goto free_sgt_table;
+	}
+
+	pr_debug("va:0x%lx len:0x%lx npages:%d nents:%u page_aligned_va:0x%lx page_offset:0x%lx\n",
+		 va, len, npages, sgt->nents, va & PAGE_MASK, va & ~PAGE_MASK);
+
+	md = cxi_map_sgtable_vf(lni, sgt, flags);
+	if (IS_ERR(md)) {
+		rc = PTR_ERR(md);
+		goto unmap_sgt;
+	}
+
+	md_priv_vf = container_of(md, struct cxi_md_priv_vf, md);
+	md_priv_vf->md.va = va;
+	md_priv_vf->sgt = sgt;
+	md_priv_vf->pages = pages;
+	md_priv_vf->npages = npages;
+
+	/* Adjust IOVA to include the within-page offset.
+	 * PF returns page-aligned IOVA, but application needs the actual data address.
+	 */
+	md_priv_vf->md.iova += (va & ~PAGE_MASK);
+	md_priv_vf->md.len = len;
+
+	return md;
+
+unmap_sgt:
+	dma_unmap_sgtable(&hw->cdev.pdev->dev, sgt, DMA_BIDIRECTIONAL, 0);
+free_sgt_table:
+	sg_free_table(sgt);
+free_sgt:
+	kfree(sgt);
+unpin_pages:
+	if (flags & CXI_MAP_PIN)
+		unpin_user_pages(pages, npages);
+free_pages:
+	kvfree(pages);
+	return ERR_PTR(rc);
+}
+
+/**
+ * cxi_map() - Map virtual addresses into IO  address space
+ *
+ * If CXI_MAP_ALLOC_MD is set, just allocate an IOVA and return the MD.
+ *
+ * @lni: the Logical Network Interface
+ * @va: kernel or user virtual address
+ * @len: length of address to map in bytes
+ * @flags: various options affecting the map
+ * @hints: hints used for debugging
+ *
+ * @return: memory descriptor or error code
+ */
+struct cxi_md *cxi_map(struct cxi_lni *lni, uintptr_t va, size_t len,
+		       u32 flags, const struct cxi_md_hints *hints)
+{
+	struct cxi_lni_priv *lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	int ret;
+	struct cxi_md_priv *md_priv;
+	struct cxi_md *md;
+	struct ac_map_opts m_opts = {
+		.huge_shift = PMD_SHIFT,
+		.page_shift = PAGE_SHIFT
+	};
+	struct cass_ac *cac;
+	u64 ova = va;
+	size_t olen = len;
+
+	if (!len) {
+		pr_debug("Length is 0\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (flags & CXI_MAP_PIN &&
+	    (flags & CXI_MAP_FAULT || flags & CXI_MAP_PREFETCH)) {
+		pr_debug("Invalid flags combination:0x%x\n", flags);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (hints && (hints->ptg_mode_valid)) {
+		/* Invalid combination. hugepage_test builds two
+		 * level tables.
+		 */
+		if ((hints->ptg_mode == C_ATU_PTG_MODE_SGL) &&
+		    m_opts.hugepage_test) {
+			cxidev_info(cdev, "Hugepage test cannot be used with PTG Mode Single\n");
+			return ERR_PTR(-EINVAL);
+		}
+
+		if ((hints->ptg_mode < C_ATU_PTG_MODE_SGL) ||
+		    (hints->ptg_mode > C_ATU_PTG_MODE_DBL_C)) {
+			cxidev_info(cdev, "Invalid PTG_MODE supplied:%d\n",
+				    hints->ptg_mode);
+			return ERR_PTR(-EINVAL);
+		}
+		m_opts.ptg_mode = hints->ptg_mode;
+	} else {
+		m_opts.ptg_mode = default_ptg_mode;
+	}
+
+	if (!cdev->is_physfn)
+		return cxi_map_vf(lni, va, len, flags, hints);
+
+	ret = cass_odp_supported(hw, flags);
+	if (ret)
+		return ERR_PTR(ret);
+
+	md_priv = kzalloc(sizeof(*md_priv), GFP_KERNEL);
+	if (md_priv == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	refcount_set(&md_priv->refcount, 1);
+	md = &md_priv->md;
+	md_priv->lni_priv = lni_priv;
+	md_priv->device = &hw->cdev.pdev->dev;
+	m_opts.flags = flags;
+	m_opts.md_priv = md_priv;
+
+	if (flags & CXI_MAP_ATS) {
+		cass_align_start_len(&m_opts, va, len, PAGE_SHIFT);
+	} else if (flags & CXI_MAP_DEVICE) {
+		if (hints) {
+			if (hints->dmabuf_valid) {
+				md_priv->dmabuf_fd = hints->dmabuf_fd;
+				md_priv->dmabuf_offset = hints->dmabuf_offset;
+				md_priv->dmabuf_length = olen;
+			} else {
+				md_priv->dmabuf_fd = INVALID_DMABUF_FD;
+			}
+		}
+
+		ret = cass_is_device_memory(hw, &m_opts, va, len);
+		if (ret)
+			goto md_free;
+
+		cass_align_start_len(&m_opts, va, len, m_opts.page_shift);
+
+		ret = cass_device_get_pages(&m_opts);
+		if (ret)
+			goto md_free;
+
+	} else if (flags & CXI_MAP_USER_ADDR) {
+		int align_shift = m_opts.page_shift;
+
+		m_opts.va_end = va + len;
+		ret = cass_cpu_page_size(hw, &m_opts, current->mm, va, hints,
+					 &align_shift);
+		if (ret)
+			goto md_free;
+
+		cass_align_start_len(&m_opts, va, len, align_shift);
+	} else {
+		/* VA is a kernel address which is expected to be aligned
+		 * to the PAGE_SIZE.
+		 */
+		if (va & ~PAGE_MASK) {
+			cxidev_err(cdev, "va not aligned 0x%lx\n", va);
+			ret = -EINVAL;
+			goto md_free;
+		}
+
+		cass_align_start_len(&m_opts, va, len, m_opts.page_shift);
+	}
+
+	if (m_opts.ptg_mode == C_ATU_PTG_MODE_SGL)
+		m_opts.huge_shift = m_opts.page_shift;
+
+	cac = get_matching_ac(lni_priv, &m_opts);
+	if (IS_ERR(cac)) {
+		cxidev_dbg(cdev, "Failure to get AC:%ld\n", PTR_ERR(cac));
+		ret = PTR_ERR(cac);
+		goto put_device_pages;
+	}
+
+	md_priv->cac = cac;
+
+	/* Get an MD ID, above CXI_MD_NONE */
+	ret = ida_alloc_min(&hw->md_index_table, 1, GFP_KERNEL);
+	if (ret < 0) {
+		cxidev_err(cdev, "ida_alloc_min failed %d\n", ret);
+		goto iova_free;
+	}
+
+	md->id = ret;
+
+	cass_init_md(&m_opts, md_priv);
+
+	pr_debug("ova:%llx olen:%lx va:%llx iova:%0llx iova_base:%llx md:%d ac:%d len:%lx flags:%x\n",
+		 ova, olen, m_opts.va_start, md->iova, cac->iova_base,
+		 md->id, md_priv->cac->ac.acid, md->len, flags);
+
+	if (!(flags & CXI_MAP_ALLOC_MD)) {
+		ret = cass_mirror_range(md_priv, &m_opts);
+		if (ret) {
+			pr_debug("Mirror address range failed:%d\n", ret);
+			goto mirror_range_error;
+		}
+	}
+
+	cass_add_md(lni_priv, &m_opts, md_priv);
+	refcount_inc(&lni_priv->refcount);
+
+	return md;
+
+mirror_range_error:
+	ida_free(&hw->md_index_table, md->id);
+iova_free:
+	cass_md_iova_free(md_priv);
+put_device_pages:
+	if (flags & CXI_MAP_DEVICE)
+		cass_device_put_pages(md_priv);
+md_free:
+	kfree(md_priv);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(cxi_map);
+
+static int cxi_unmap_vf(struct cxi_md *md)
+{
+	struct cxi_md_priv_vf *md_priv_vf = container_of(md, struct cxi_md_priv_vf, md);
+	struct cxi_dev *dev = md_priv_vf->lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+
+	pr_debug("VF md:%d lac:%d va:%llx iova:%llx len:%lx\n",
+		 md->id, md->lac, md->va, md->iova, md->len);
+
+	/* Notify PF to free the MD created from the VF sglist */
+	cxi_unmap_sgtable_vf(dev, md_priv_vf, hw, md->id);
+
+	/* Unmap and free scatter-gather table if we own it */
+	if (md_priv_vf->pages && md_priv_vf->sgt) {
+		dma_unmap_sgtable(md_priv_vf->device, md_priv_vf->sgt, DMA_BIDIRECTIONAL, 0);
+		sg_free_table(md_priv_vf->sgt);
+		kfree(md_priv_vf->sgt);
+	}
+
+	/* Unpin pages if they were pinned, or just free the pages array */
+	if (md_priv_vf->pages) {
+		if (md_priv_vf->flags & CXI_MAP_PIN)
+			unpin_user_pages(md_priv_vf->pages, md_priv_vf->npages);
+		kvfree(md_priv_vf->pages);
+	}
+
+	/* Free the VF MD structure */
+	kfree(md_priv_vf);
+
+	return 0;
+}
+
+/**
+ * cxi_unmap() - Unmap a virtual address
+ *
+ * @md: Memory Descriptor used in network operations
+ */
+int cxi_unmap(struct cxi_md *md)
+{
+	struct cxi_md_priv *md_priv;
+	struct cxi_lni_priv *lni_priv;
+	struct cxi_dev *dev;
+	struct cass_dev *hw;
+
+	if (!md || md->lac >= C_NUM_LACS)
+		return -EINVAL;
+
+	dev = container_of(md, struct cxi_md_priv, md)->lni_priv->dev;
+	if (!dev->is_physfn)
+		return cxi_unmap_vf(md);
+
+	md_priv = container_of(md, struct cxi_md_priv, md);
+	lni_priv = md_priv->lni_priv;
+	dev = lni_priv->dev;
+	hw = container_of(dev, struct cass_dev, cdev);
+
+	/* One reference for CAC allocation, one reference for MD. */
+	if (refcount_read(&md_priv->cac->refcount) < 2)
+		return -EINVAL;
+
+	pr_debug("md:%d ac:%d lac:%d va:%llx iova:%llx len:%lx\n",
+		 md->id, md_priv->cac->ac.acid, md->lac, md->va, md->iova,
+		 md->len);
+
+	cass_remove_md(md_priv);
+	cass_md_iova_free(md_priv);
+	ida_free(&hw->md_index_table, md->id);
+
+	cxidev_WARN_ONCE(dev, !refcount_dec_and_test(&md_priv->refcount),
+			 "Resource leaks - MD refcount not zero: %d\n",
+			 refcount_read(&md_priv->refcount));
+
+	cass_dma_unmap_pages(md_priv);
+	kfree(md_priv);
+
+	refcount_dec(&lni_priv->refcount);
+
+	return 0;
+}
+EXPORT_SYMBOL(cxi_unmap);
+
+/**
+ * cxi_update_md() - Update the specified address range contained in MD
+ *
+ * @md: Memory Descriptor used in network operations
+ * @va: User virtual address
+ * @len: Length of address to fault in bytes
+ * @flags: Currently only supports faulting pages with CXI_MAP_FAULT flag
+ *         for an ODP MD.
+ *
+ * @return: 0 on success or negative error value
+ */
+int cxi_update_md(struct cxi_md *md, uintptr_t va, size_t len, u32 flags)
+{
+	int ret;
+	struct cxi_md_priv *md_priv;
+	struct cass_dev *hw;
+	int align_shift;
+	struct ac_map_opts m_opts = {};
+
+	if (!md || md->lac >= C_NUM_LACS)
+		return -EINVAL;
+
+	if (va < md->va || len > ULONG_MAX - va ||
+	    va - md->va > md->len || len > md->len - (va - md->va)) {
+		pr_debug("Address range not bounded by MD\n");
+		return -EINVAL;
+	}
+
+	/* Just support CXI_MAP_FAULT for now */
+	if (!(flags & CXI_MAP_FAULT)) {
+		pr_debug("Only CXI_MAP_FAULT is supported\n");
+		return -EINVAL;
+	}
+
+	md_priv = container_of(md, struct cxi_md_priv, md);
+	hw = container_of(md_priv->lni_priv->dev, struct cass_dev, cdev);
+
+	if (md_priv->flags & CXI_MAP_PIN) {
+		pr_debug("Must be ODP MD\n");
+		return -EINVAL;
+	}
+
+	m_opts.va_end = va + len;
+	m_opts.md_priv = md_priv;
+	m_opts.page_shift = md->page_shift;
+	m_opts.huge_shift = md->huge_shift;
+	/* Use the permissions from the MD */
+	m_opts.flags = md_priv->flags & (CXI_MAP_READ | CXI_MAP_WRITE);
+	m_opts.flags |= flags;
+	align_shift = m_opts.page_shift;
+
+	ret = cass_cpu_page_size(hw, &m_opts, current->mm, va, NULL,
+				 &align_shift);
+	if (ret)
+		return ret;
+
+	/* We only support VMAs that match the MD's hugepage size. */
+	if (m_opts.huge_shift != md->huge_shift) {
+		pr_debug("Error: VMA huge_shift (%d) must match MD (%d)\n",
+			 m_opts.huge_shift, md->huge_shift);
+		return -EINVAL;
+	}
+
+	cass_align_start_len(&m_opts, va, len, align_shift);
+	m_opts.iova = md->iova + (m_opts.va_start - md->va);
+
+	pr_debug("md:%d ac:%d md.va:%llx va:%llx iova:%llx len:%lx flags:%x\n",
+		 md->id, md_priv->cac->ac.acid, md->va, m_opts.va_start,
+		 m_opts.iova, m_opts.va_len, m_opts.flags);
+
+	ret = cass_mirror_odp(&m_opts, md_priv->cac,
+			      m_opts.va_len >> m_opts.page_shift,
+			      m_opts.va_start);
+
+	return ret;
+}
+EXPORT_SYMBOL(cxi_update_md);

@@ -0,0 +1,1243 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2018, 2024-2026 Hewlett Packard Enterprise Development LP */
+
+/* Create and destroy Cassini command queues */
+
+#include <linux/hpe/cxi/cxi.h>
+#include <linux/dma-mapping.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+#include "cass_core.h"
+#include "cxi_internal.h"
+#include "cass_ss1_debugfs.h"
+#include "cxi_vf_cmd.h"
+
+#ifdef CONFIG_ARM64
+/* Provide a workaround for avoiding writecombine on platforms where it is broken
+ * and for which the Linux kernel does not already provide a workaround, such as the
+ * Ampere Altra, used in the RL300.
+ */
+DEFINE_STATIC_KEY_FALSE(avoid_writecombine);
+#endif
+
+/* Lower bound for number of 64 byte commands. */
+#define MIN_CQ_COUNT (PAGE_SIZE / C_CQ_CMD_SIZE)
+
+void cass_cq_init(struct cass_dev *hw)
+{
+	static const union c_cq_cfg_cq_policy cq_policy = {
+		.policy[CXI_CQ_UPDATE_ALWAYS] = {
+			.shift = 0,
+		},
+		.policy[CXI_CQ_UPDATE_HIGH_FREQ_EMPTY] = {
+			.shift = 4,
+			.empty = 1,
+		},
+		.policy[CXI_CQ_UPDATE_LOW_FREQ_EMPTY] = {
+			.shift = 8,
+			.empty = 1,
+		},
+		.policy[CXI_CQ_UPDATE_LOW_FREQ] = {
+			.shift = 8,
+		},
+	};
+	union c_cq_cfg_tg_thresh tg_thresh = {
+		.tg[0].thresh = tg_threshold[0],
+		.tg[1].thresh = tg_threshold[1],
+		.tg[2].thresh = tg_threshold[2],
+		.tg[3].thresh = tg_threshold[3],
+	};
+
+	/* Initialize CQ policies. */
+	cass_write(hw, C_CQ_CFG_CQ_POLICY, &cq_policy, sizeof(cq_policy));
+
+	/* LPE append credits */
+	cass_write(hw, C_CQ_CFG_TG_THRESH, &tg_thresh, sizeof(tg_thresh));
+
+	/* Zero FQ allocation settings. */
+	cass_clear(hw, C_CQ_CFG_FQ_RESRV(0), C_CQ_CFG_FQ_RESRV_SIZE);
+
+#ifdef CONFIG_ARM64
+	if (pci_get_device(PCI_VENDOR_ID_AMPERE, 0xe100, NULL))
+		static_branch_enable(&avoid_writecombine);
+#endif
+}
+
+static phys_addr_t
+cq_mmio_phys_addr(const struct cass_dev *hw, int cq_id)
+{
+	return hw->regs_base + C_CQ_LAUNCH_TXQ_BASE +
+		C_CQ_LAUNCH_PAGE_SIZE * cq_id;
+}
+
+/* Write TX CQ configuration to hardware */
+static int setup_hw_tx_cq(struct cass_dev *hw, unsigned int cq_n,
+			  const struct cxi_cq_alloc_opts *opts,
+			  struct cxi_lni_priv *lni_priv,
+			  struct cxi_cq_priv *cq,
+			  struct cxi_eq_priv *eq,
+			  struct cxi_cp_priv *cp_priv)
+{
+	size_t cmds_count = cq->cmds_len / C_CQ_CMD_SIZE;
+	union c_cq_txq_tc_table tc = {
+		.mem_q_eth = !!(opts->flags & CXI_CQ_TX_ETHERNET),
+		.mem_q_trig = !!(opts->flags & CXI_CQ_TX_WITH_TRIG_CMDS),
+	};
+	union c_cq_txq_base_table txq_cfg = {
+		.mem_q_eq = eq ? eq->eq.eqn : C_EQ_NONE,
+		.mem_q_base = cq->cmds_dma_addr >> C_ADDR_SHIFT,
+		.mem_q_max_ptr = cmds_count - 1,
+		.mem_q_rgid = lni_priv->lni.rgid,
+		.mem_q_stat_cnt_pool = opts->stat_cnt_pool,
+		.mem_q_lcid = opts->lcid,
+		.mem_q_acid = lni_priv->phys_ac,
+		.mem_q_policy = opts->policy,
+	};
+	union c_cq_cfg_init_txq_hw_state txq_init;
+	union c_cq_cfg_cp_fl_table flow_cfg;
+	void __iomem *csr;
+	int rc;
+
+	/* Transmit CQ needs to be configured with the same PFQ and TC
+	 * values the corresponding communication profile is using.
+	 */
+	cass_read(hw, C_CQ_CFG_CP_FL_TABLE(cp_priv->cass_cp->id),
+		  &flow_cfg, sizeof(flow_cfg));
+
+	tc.mem_q_tc = flow_cfg.tc;
+	txq_cfg.mem_q_pfq = flow_cfg.pfq;
+
+	if (opts->flags & CXI_CQ_TX_WITH_TRIG_CMDS)
+		txq_cfg.mem_q_pfq = cxi_rgroup_tle_pool_id(lni_priv->rgroup);
+
+	spin_lock(&hw->cq_shadow_lock);
+	cass_tx_cq_init(hw, &cq->cass_cq, cq_n, &tc, &txq_cfg);
+	spin_unlock(&hw->cq_shadow_lock);
+
+	mutex_lock(&hw->cq_init_lock);
+	cass_read(hw, C_CQ_CFG_INIT_TXQ_HW_STATE, &txq_init,
+		  sizeof(txq_init));
+
+	cxidev_WARN_ONCE(&hw->cdev, txq_init.pending == 1,
+			 "TXQ init pending CSR should not be set");
+	if (txq_init.pending == 1) {
+		mutex_unlock(&hw->cq_init_lock);
+		return -EIO;
+	}
+
+	txq_init.cq_handle = cq_n;
+
+	cass_write(hw, C_CQ_CFG_INIT_TXQ_HW_STATE, &txq_init,
+		   sizeof(txq_init));
+
+	csr = cass_csr(hw, C_CQ_CFG_INIT_TXQ_HW_STATE);
+	rc = readq_poll_timeout(csr, txq_init.qw, txq_init.pending == 0,
+				1, 1000);
+
+	cxidev_WARN_ONCE(&hw->cdev, rc, "Timeout waiting for TXQ init");
+	if (rc) {
+		mutex_unlock(&hw->cq_init_lock);
+		return -EIO;
+	}
+	mutex_unlock(&hw->cq_init_lock);
+
+	cass_tx_cq_enable(hw, cq_n);
+
+	return 0;
+}
+
+/* Write TGT CQ configuration to hardware */
+#define MEM_Q_TC_REG_SIZE BIT(3)
+static int setup_hw_tgt_cq(struct cass_dev *hw, unsigned int cq_n,
+			   const struct cxi_cq_alloc_opts *opts,
+			   struct cxi_lni_priv *lni_priv,
+			   struct cxi_cq_priv *cq,
+			   struct cxi_eq_priv *eq)
+{
+	size_t cmds_count = cq->cmds_len / C_CQ_CMD_SIZE;
+	// TODO: Handle TGT TC
+	const union c_cq_tgq_table tgt_cfg = {
+		.mem_q_eq = eq ? eq->eq.eqn : C_EQ_NONE,
+		.mem_q_base = cq->cmds_dma_addr >> C_ADDR_SHIFT,
+		.mem_q_max_ptr = cmds_count - 1,
+		.mem_q_rgid = lni_priv->lni.rgid,
+		.mem_q_stat_cnt_pool = opts->stat_cnt_pool,
+		.mem_q_tc_reg = cq_n % MEM_Q_TC_REG_SIZE,
+		.mem_q_acid = lni_priv->phys_ac,
+		.mem_q_policy = opts->policy,
+	};
+	union c_cq_cfg_init_tgq_hw_state tgq_init;
+	union c_cq_cfg_thresh_map thresh_map = {
+		.thresh_id = opts->lpe_cdt_thresh_id,
+	};
+	void __iomem *csr;
+	int rc;
+
+	spin_lock(&hw->cq_shadow_lock);
+	cass_tgt_cq_init(hw, &cq->cass_cq, cq_n, &tgt_cfg);
+	spin_unlock(&hw->cq_shadow_lock);
+
+	mutex_lock(&hw->cq_init_lock);
+	cass_read(hw, C_CQ_CFG_INIT_TGQ_HW_STATE, &tgq_init,
+		  sizeof(tgq_init));
+
+	cxidev_WARN_ONCE(&hw->cdev, tgq_init.pending == 1,
+			 "TGQ init pending CSR should not be set");
+	if (tgq_init.pending == 1) {
+		mutex_unlock(&hw->cq_init_lock);
+		return -EIO;
+	}
+
+	cass_write(hw, C_CQ_CFG_THRESH_MAP(cq_n),
+		   &thresh_map, sizeof(thresh_map));
+
+	tgq_init.cq_handle = cq_n;
+	cass_write(hw, C_CQ_CFG_INIT_TGQ_HW_STATE, &tgq_init,
+		   sizeof(tgq_init));
+
+	/* TODO: Check delay and timeouts are correct for real NIC. */
+	csr = cass_csr(hw, C_CQ_CFG_INIT_TGQ_HW_STATE);
+	rc = readq_poll_timeout(csr, tgq_init.qw, tgq_init.pending == 0,
+				1, 1000);
+
+	cxidev_WARN_ONCE(&hw->cdev, rc, "Timeout waiting for TGQ init");
+	if (rc) {
+		mutex_unlock(&hw->cq_init_lock);
+		return -EIO;
+	}
+	mutex_unlock(&hw->cq_init_lock);
+
+	cass_tgt_cq_enable(hw, cq_n);
+
+	return 0;
+}
+
+/* Return an available CQ with an ID */
+static struct cxi_cq_priv *pf_get_cq_id(struct cxi_lni_priv *lni_priv,
+					const struct cxi_cq_alloc_opts *opts)
+{
+	struct cxi_dev *dev = lni_priv->dev;
+	struct cass_dev *hw =
+		container_of(dev, struct cass_dev, cdev);
+	struct cxi_cq_priv *cq;
+	enum cxi_resource_type rsrc_type;
+	int rc;
+
+	cq = kzalloc(sizeof(struct cxi_cq_priv), GFP_KERNEL);
+	if (cq == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	/* Check the associated service to see if this CQ can be allocated */
+	if (opts->flags & CXI_CQ_IS_TX)
+		rsrc_type = CXI_RESOURCE_TXQ;
+	else
+		rsrc_type = CXI_RESOURCE_TGQ;
+
+	rc = cxi_rgroup_alloc_resource(lni_priv->rgroup, rsrc_type);
+	if (rc) {
+		/* Attempt to cleanup all CQs on this LNI to return service
+		 * credits and then reattempt to allocate resource.
+		 */
+		finalize_cq_cleanups(lni_priv, false);
+
+		rc = cxi_rgroup_alloc_resource(lni_priv->rgroup, rsrc_type);
+		if (rc)
+			goto cq_free;
+	}
+
+	if (opts->flags & CXI_CQ_IS_TX)
+		rc = ida_alloc_range(&hw->cq_table, 0, C_NUM_TRANSMIT_CQS - 1, GFP_KERNEL);
+	else
+		rc = ida_alloc_range(&hw->cq_table, C_NUM_TRANSMIT_CQS,
+				     C_NUM_TRANSMIT_CQS + C_NUM_TARGET_CQS - 1,
+				     GFP_KERNEL);
+
+	if (rc < 0) {
+		/* Out of CQs. Accelerate possible CQ cleanup. */
+		lni_cleanups(hw, false);
+
+		if (opts->flags & CXI_CQ_IS_TX)
+			rc = ida_alloc_range(&hw->cq_table, 0,
+					     C_NUM_TRANSMIT_CQS - 1, GFP_KERNEL);
+		else
+			rc = ida_alloc_range(&hw->cq_table, C_NUM_TRANSMIT_CQS,
+					     C_NUM_TRANSMIT_CQS + C_NUM_TARGET_CQS - 1,
+					     GFP_KERNEL);
+	}
+
+	if (rc < 0)
+		goto dec_rsrc_use;
+
+	cq->cass_cq.idx = rc;
+
+	return cq;
+
+dec_rsrc_use:
+	cxi_rgroup_free_resource(lni_priv->rgroup, rsrc_type);
+
+cq_free:
+	kfree(cq);
+
+	return ERR_PTR(rc);
+}
+
+static void put_cq_id(struct cxi_cq_priv *cq)
+{
+	struct cxi_lni_priv *lni_priv = cq->lni_priv;
+	struct cxi_dev *dev = lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+
+	ida_free(&hw->cq_table, cq->cass_cq.idx);
+	if (cq->flags & CXI_CQ_IS_TX)
+		cxi_rgroup_free_resource(lni_priv->rgroup, CXI_RESOURCE_TXQ);
+	else
+		cxi_rgroup_free_resource(lni_priv->rgroup, CXI_RESOURCE_TGQ);
+	kfree(cq);
+}
+
+/**
+ * cxi_cq_free_vf() - Free the communication profile for VF
+ *
+ * @cmdq: Command queue to be freed
+ */
+static void cxi_cq_free_vf(struct cxi_cq *cmdq)
+{
+	struct cxi_cq_priv_vf *cq_priv_vf =
+		container_of(cmdq, struct cxi_cq_priv_vf, cass_cq);
+	struct cxi_dev *dev = cq_priv_vf->lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	const struct cxi_cq_free_cmd cmd = {
+		.op = CXI_OP_CQ_FREE,
+		.cq = cq_priv_vf->cass_cq.idx,
+	};
+	size_t resp_len = 0;
+	int rc;
+
+	rc = cxi_send_msg_to_pf(dev, &cmd, sizeof(cmd), NULL, &resp_len);
+	if (rc)
+		return;
+
+	if (cq_priv_vf->cq_mmio)
+		iounmap(cq_priv_vf->cq_mmio);
+
+	dma_unmap_page(&hw->cdev.pdev->dev, cq_priv_vf->cmds_dma_addr,
+		       cq_priv_vf->cmds_len, DMA_BIDIRECTIONAL);
+
+	if (cq_priv_vf->cmds_pages) {
+		__free_pages(cq_priv_vf->cmds_pages, cq_priv_vf->cmds_order);
+	} else {
+		if (cq_priv_vf->md_priv) {
+			cass_device_put_pages(cq_priv_vf->md_priv);
+			kfree(cq_priv_vf->md_priv);
+		}
+
+		if (cq_priv_vf->pages) {
+			unpin_user_pages(cq_priv_vf->pages, cq_priv_vf->cmds_len >> PAGE_SHIFT);
+			kfree(cq_priv_vf->pages);
+		}
+	}
+
+	kfree(cq_priv_vf);
+}
+
+/**
+ * get_dma_addr_if_contig() - Check for contiguous pages of user buffer,
+ *                            pin and return dma_addr.
+ *
+ * If pages are contiguous, pin the pages and return the dma_addr of the base.
+ * Otherwise, return negative errno.
+ *
+ * @hw: The cassini device
+ * @addr: address of page aligned buffer
+ * @len: page aligned length of buffer - must not be 0
+ * @dir: dma map direction
+ * @pages: ptr to list of pages returned
+ * @dma_addr: dma_addr of user buffer
+ *
+ * @return: 0 on success, negative error value
+ */
+int get_dma_addr_if_contig(struct cass_dev *hw, void *addr, size_t len,
+			   enum dma_data_direction dir, struct page ***pages,
+			   dma_addr_t *dma_addr)
+{
+	int i;
+	int rc;
+	int npinned;
+	dma_addr_t pa;
+	size_t npg = len >> PAGE_SHIFT;
+
+	*pages = kmalloc_array(npg, sizeof(struct page *), GFP_KERNEL);
+	if (!*pages)
+		return -ENOMEM;
+
+	npinned = pin_user_pages_fast((u64)addr, npg,
+				      FOLL_WRITE | FOLL_LONGTERM, *pages);
+	if (npinned < 0) {
+		rc = npinned;
+		goto free_pages;
+	}
+
+	if (npinned != npg) {
+		rc = -EFAULT;
+		goto put_pages;
+	}
+
+	/* Naively check that each page is in sequence. */
+	pa = page_to_phys((*pages)[0]);
+	for (i = 1; i < npg; i++) {
+		pa += PAGE_SIZE;
+		if (pa != page_to_phys((*pages)[i])) {
+			cxidev_dbg(&hw->cdev,
+				   "Buffer supplied is required to be contiguous\n");
+			rc = -EINVAL;
+			goto put_pages;
+		}
+	}
+
+	*dma_addr = dma_map_page(&hw->cdev.pdev->dev, (*pages)[0], 0, len, dir);
+	if (dma_mapping_error(&hw->cdev.pdev->dev, *dma_addr)) {
+		rc = -ENOMEM;
+		goto put_pages;
+	}
+
+	return 0;
+
+put_pages:
+	unpin_user_pages(*pages, npinned);
+
+free_pages:
+	kfree(*pages);
+
+	return rc;
+}
+
+static struct cxi_md_priv *get_device_addr(struct cxi_lni_priv *lni_priv,
+					   uintptr_t va, size_t len,
+					   int dmabuf_fd,
+					   unsigned long dmabuf_offset,
+					   dma_addr_t *dma_addr)
+{
+	int i;
+	int rc;
+	struct scatterlist *sg;
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	struct cxi_md_priv *md_priv;
+	struct ac_map_opts m_opts = {
+		.huge_shift = PMD_SHIFT,
+		.page_shift = PAGE_SHIFT
+	};
+	u32 flags = CXI_MAP_DEVICE;
+	dma_addr_t next_addr = 0;
+
+	if (!len) {
+		pr_debug("Length is 0\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	md_priv = kzalloc(sizeof(*md_priv), GFP_KERNEL);
+	if (!md_priv)
+		return ERR_PTR(-ENOMEM);
+
+	refcount_set(&md_priv->refcount, 1);
+	md_priv->lni_priv = lni_priv;
+	md_priv->device = &hw->cdev.pdev->dev;
+	md_priv->dmabuf_fd = dmabuf_fd;
+	md_priv->dmabuf_offset = dmabuf_offset;
+	md_priv->dmabuf_length = len;
+	m_opts.flags = flags;
+	m_opts.md_priv = md_priv;
+
+	rc = cass_is_device_memory(hw, &m_opts, va, len);
+	if (rc)
+		goto md_free;
+
+	cass_align_start_len(&m_opts, va, len, m_opts.page_shift);
+
+	rc = cass_device_get_pages(&m_opts);
+	if (rc) {
+		cxidev_dbg(&hw->cdev,
+			   "cass_device_get_pages failed rc:%d\n", rc);
+		goto md_free;
+	}
+
+	/* AMD p2p pages are not guaranteed to be contiguous. */
+	for_each_sgtable_dma_sg(md_priv->sgt, sg, i) {
+		long len = sg_dma_len(sg);
+		dma_addr_t dma_addr = sg_dma_address(sg);
+
+		if (dma_addr != PAGE_ALIGN(dma_addr)) {
+			cxidev_dbg(&hw->cdev,
+				   "Buffer supplied is required to be page aligned\n");
+			rc = -EINVAL;
+			goto put_pages;
+		}
+
+		if (!next_addr) {
+			next_addr = dma_addr + len;
+			continue;
+		}
+
+		if (dma_addr != next_addr) {
+			cxidev_dbg(&hw->cdev,
+				   "Buffer supplied is required to be contiguous\n");
+			rc = -EINVAL;
+			goto put_pages;
+		}
+
+		next_addr = dma_addr + len;
+	}
+
+	md_priv->olen = m_opts.va_len;
+	md_priv->flags = m_opts.flags;
+	*dma_addr = sg_dma_address(md_priv->sgt->sgl);
+
+	return md_priv;
+
+put_pages:
+	cass_device_put_pages(md_priv);
+
+md_free:
+	kfree(md_priv);
+
+	return ERR_PTR(rc);
+}
+
+/**
+ * cxi_cq_alloc_buf_vf() - Allocate a new command queue for VF
+ *
+ * The new CQ is attached to attached to the NI.
+ *
+ * @lni: LNI to associate with the command queue.
+ * @evtq: Event Queue to associate with the command queue, for error
+ *        reporting. May be NULL.
+ * @opts_b: Options affecting the CQ.
+ * @numa_node: NUMA node ID CQ memory should be allocated from.
+ *
+ * @return: the command queue or an error pointer
+ */
+static struct cxi_cq *cxi_cq_alloc_buf_vf(struct cxi_lni *lni, struct cxi_eq *evtq,
+					  const struct cxi_cq_alloc_opts_buf *opts_b,
+					  int numa_node)
+{
+	struct cxi_lni_priv_vf *lni_priv_vf =
+		container_of(lni, struct cxi_lni_priv_vf, lni);
+	struct cxi_dev *cdev = lni_priv_vf->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	const struct cxi_cq_alloc_opts *opts = &opts_b->opts;
+	struct cxi_cq_priv_vf *cq;
+	struct cxi_cq_alloc_buf_cmd_vf cmd = {
+		.base.op = CXI_OP_CQ_ALLOC_BUF,
+		.base.lni = lni_priv_vf->lni.id,
+		.base.eq = evtq ? evtq->eqn : C_EQ_NONE,
+		.base.opts = *opts_b,
+	};
+	struct cxi_cq_alloc_resp resp;
+	size_t resp_len = sizeof(resp);
+	bool is_user = opts->flags & CXI_CQ_USER;
+	struct page *cmds_pages;
+	size_t cmds_count;
+	size_t cmds_order;
+	size_t cmds_len;
+	int rc;
+
+	/* Use provided commands count to calculate commands length. Then, use
+	 * commands length to recalculate commands count to consume entire
+	 * buffer space.
+	 */
+	cmds_count = max_t(size_t, MIN_CQ_COUNT, opts->count);
+	cmds_order = get_order(cmds_count * C_CQ_CMD_SIZE);
+	cmds_len = (1 << cmds_order) * PAGE_SIZE;
+	cmds_count = cmds_len / C_CQ_CMD_SIZE;
+
+	if (cmds_count > CXI_MAX_CQ_COUNT) {
+		cxidev_dbg(cdev, "cmds_count:%ld is > CXI_MAX_CQ_COUNT:%d\n",
+			   cmds_count, CXI_MAX_CQ_COUNT);
+		return ERR_PTR(-E2BIG);
+	}
+
+	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
+	if (!cq)
+		return ERR_PTR(-ENOMEM);
+
+	if (opts_b->buf) {
+		if (!IS_ALIGNED((u64)opts_b->buf, PAGE_SIZE)) {
+			cxidev_dbg(cdev, "buffer %llx not aligned to PAGE_SIZE\n",
+				   (u64)opts_b->buf);
+			rc = -EINVAL;
+			goto free_cq;
+		}
+
+		if (opts_b->is_host_mem) {
+			rc = get_dma_addr_if_contig(hw, opts_b->buf, cmds_len,
+						    DMA_BIDIRECTIONAL,
+						    &cq->pages,
+						    &cq->cmds_dma_addr);
+			if (rc)
+				goto free_cq;
+		} else {
+			cq->md_priv = get_device_addr(container_of(lni, struct cxi_lni_priv, lni),
+						      (uintptr_t)opts_b->buf,
+						      cmds_len,
+						      opts_b->dmabuf_fd,
+						      opts_b->dmabuf_offset,
+						      &cq->cmds_dma_addr);
+			if (IS_ERR(cq->md_priv)) {
+				rc = PTR_ERR(cq->md_priv);
+				goto free_cq;
+			}
+		}
+		cq->cmds = NULL;
+	} else {
+		cmds_pages = alloc_pages_node(numa_node, GFP_KERNEL | __GFP_ZERO,
+					      cmds_order);
+		if (!cmds_pages) {
+			rc = -ENOMEM;
+			goto free_cq;
+		}
+
+		cq->cmds_order = cmds_order;
+		cq->cmds = page_address(cmds_pages);
+		cq->cmds_pages = cmds_pages;
+		cq->cmds_dma_addr = dma_map_page(&hw->cdev.pdev->dev,
+						 cmds_pages, 0, cmds_len,
+						 DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(&hw->cdev.pdev->dev, cq->cmds_dma_addr)) {
+			rc = -ENOMEM;
+			goto free_pages;
+		}
+	}
+
+	cq->cmds_len = cmds_len;
+	cmd.dma_addr = cq->cmds_dma_addr;
+	cmd.cmds_len = cmds_len;
+
+	rc = cxi_send_msg_to_pf(cdev, &cmd, sizeof(cmd), &resp, &resp_len);
+	if (rc)
+		goto cq_release;
+
+	if (!is_user) {
+		phys_addr_t mmio_phys = cq_mmio_phys_addr(hw, resp.cq);
+#ifdef CONFIG_ARM64
+		if (static_branch_unlikely(&avoid_writecombine))
+			cq->cq_mmio = ioremap(mmio_phys, PAGE_SIZE);
+		else
+#endif
+		cq->cq_mmio = ioremap_wc(mmio_phys, PAGE_SIZE);
+		if (!cq->cq_mmio) {
+			cxidev_warn_once(cdev, "ioremap_wc failed\n");
+			rc = -EFAULT;
+			goto cq_release;
+		}
+	}
+
+	/* Initialize client interface structure */
+	cxi_cq_init(&cq->cass_cq,
+		    cq->cmds,
+		    cmds_count,
+		    (__force void *)cq->cq_mmio,
+		    resp.cq);
+
+	cq->lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	cq->flags = opts->flags;
+	cq->cass_cq.size = resp.count;
+	cq->cass_cq.idx = resp.cq;
+
+	return &cq->cass_cq;
+
+cq_release:
+	dma_unmap_page(&hw->cdev.pdev->dev, cq->cmds_dma_addr,
+		       cq->cmds_len, DMA_BIDIRECTIONAL);
+
+	if (cq->pages) {
+		unpin_user_pages(cq->pages, cq->cmds_len >> PAGE_SHIFT);
+		kfree(cq->pages);
+	}
+free_pages:
+	if (cq->cmds_pages)
+		__free_pages(cq->cmds_pages, cmds_order);
+free_cq:
+	kfree(cq);
+	return ERR_PTR(rc);
+}
+
+/**
+ * cxi_cq_alloc_buf_internal() - Allocate a new command queue
+ *
+ * The new CQ is attached to the LNI.
+ * The CQ buffer may be supplied by the user. It must be physically
+ * contiguous.
+ *
+ * @lni: LNI to associate with the command queue.
+ * @evtq: Event Queue to associate with the command queue, for error
+ *        reporting. May be NULL.
+ * @opts_b: Options affecting the CQ.
+ * @numa_node: NUMA node ID CQ memory should be allocated from.
+ * @cmds_dma_addr: Pre-allocated DMA address (0 to allocate new)
+ * @cmds_len: Length of pre-allocated memory (0 to allocate new)
+ *
+ * @return: the command queue or an error pointer
+ */
+struct cxi_cq *cxi_cq_alloc_buf_internal(struct cxi_lni *lni, struct cxi_eq *evtq,
+					 const struct cxi_cq_alloc_opts_buf *opts_b,
+					 int numa_node, dma_addr_t cmds_dma_addr,
+					 size_t cmds_len)
+{
+	struct cxi_lni_priv *lni_priv =
+		container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	const struct cxi_cq_alloc_opts *opts = &opts_b->opts;
+	struct cxi_eq_priv *eq;
+	struct cxi_cq_priv *cq;
+	int cq_idx;
+	int cq_n;
+	int rc;
+	size_t cmds_count;
+	size_t cmds_order;
+	struct page *cmds_pages;
+	struct cxi_cp_priv *cp_priv = NULL;
+	bool is_user = opts->flags & CXI_CQ_USER;
+
+	if (opts->policy < 0 || opts->policy > CXI_CQ_UPDATE_LOW_FREQ)
+		return ERR_PTR(-EINVAL);
+
+	if (!(opts->flags & CXI_CQ_IS_TX) && opts->lpe_cdt_thresh_id > 3)
+		return ERR_PTR(-EINVAL);
+
+	/*
+	 * Use provided commands count to calculate commands length. Then, use
+	 * commands length to recalculate commands count to consume entire
+	 * buffer space.
+	 */
+	cmds_count = max_t(size_t, MIN_CQ_COUNT, opts->count);
+	cmds_order = get_order(cmds_count * C_CQ_CMD_SIZE);
+	cmds_len = (1 << cmds_order) * PAGE_SIZE;
+	cmds_count = cmds_len / C_CQ_CMD_SIZE;
+
+	if (cmds_count > CXI_MAX_CQ_COUNT) {
+		cxidev_dbg(cdev, "cmds_count:%ld is > CXI_MAX_CQ_COUNT:%d\n",
+			   cmds_count, CXI_MAX_CQ_COUNT);
+		return ERR_PTR(-E2BIG);
+	}
+
+	/* At least one communication profile is needed for a transmit
+	 * CQ.
+	 */
+	if (opts->flags & CXI_CQ_IS_TX) {
+		/* For triggered CQs, only LCID zero can be used. */
+		if ((opts->flags & CXI_CQ_TX_WITH_TRIG_CMDS) && opts->lcid != 0)
+			return ERR_PTR(-EINVAL);
+
+		cp_priv = cass_cp_find(hw, lni_priv->lni.rgid, opts->lcid);
+		if (!cp_priv)
+			return ERR_PTR(-EINVAL);
+	}
+
+	/* Only privileged client can use the C_CMD_ETHERNET_TX command. */
+	if ((opts->flags & CXI_CQ_TX_ETHERNET) && !capable(CAP_NET_RAW))
+		return ERR_PTR(-EPERM);
+
+	cq = pf_get_cq_id(lni_priv, opts);
+	if (IS_ERR(cq))
+		return ERR_PTR(PTR_ERR(cq));
+
+	cq_idx = cq->cass_cq.idx;
+
+	if (evtq)
+		eq = container_of(evtq, struct cxi_eq_priv, eq);
+	else
+		eq = NULL;
+
+	cq->lni_priv = lni_priv;
+	cq->eq = eq;
+	cq->flags = opts->flags;
+	cq->cmds_len = cmds_len;
+
+	if (!lni_priv->is_vf) {
+		/* For physical function, we use user buffer if provided or we
+		 * allocate a new one
+		 */
+		if (opts_b->buf) {
+			if (!IS_ALIGNED((u64)opts_b->buf, PAGE_SIZE)) {
+				cxidev_dbg(cdev, "buffer %llx not aligned to PAGE_SIZE\n",
+					   (u64)opts_b->buf);
+				rc = -EINVAL;
+				goto put_id;
+			}
+
+			if (opts_b->is_host_mem) {
+				rc = get_dma_addr_if_contig(hw, opts_b->buf, cmds_len,
+							    DMA_BIDIRECTIONAL,
+							&cq->pages,
+							&cq->cmds_dma_addr);
+				if (rc)
+					goto put_id;
+			} else {
+				cq->md_priv = get_device_addr(lni_priv,
+							      (uintptr_t)opts_b->buf,
+							cmds_len,
+							opts_b->dmabuf_fd,
+							opts_b->dmabuf_offset,
+							&cq->cmds_dma_addr);
+				if (IS_ERR(cq->md_priv)) {
+					rc = PTR_ERR(cq->md_priv);
+					goto put_id;
+				}
+			}
+
+			cq->cmds = NULL;
+		} else {
+			/* CQ needs contiguous memory region. */
+			cmds_pages = alloc_pages_node(numa_node, GFP_KERNEL | __GFP_ZERO,
+						      cmds_order);
+			if (!cmds_pages) {
+				rc = -ENOMEM;
+				goto put_id;
+			}
+
+			cq->cmds_order = cmds_order;
+			cq->cmds = page_address(cmds_pages);
+			cq->cmds_pages = cmds_pages;
+			cq->cmds_dma_addr = dma_map_page(&hw->cdev.pdev->dev,
+							 cmds_pages, 0, cq->cmds_len,
+							 DMA_BIDIRECTIONAL);
+			if (dma_mapping_error(&hw->cdev.pdev->dev, cq->cmds_dma_addr)) {
+				rc = -ENOMEM;
+				goto free_pages;
+			}
+		}
+	} else {
+		/* For VF we use the provided dma_addr */
+		if (!cmds_dma_addr) {
+			rc = -EINVAL;
+			goto put_id;
+		}
+
+		cq->cmds_len = cmds_len;
+		cq->cmds_dma_addr = cmds_dma_addr;
+		/* PF doesn't have access to VF's virtual address space.
+		 * VF has already initialized its own cxi_cq structure.
+		 */
+		cq->cmds = NULL;
+	}
+
+	if (!is_user && !lni_priv->is_vf) {
+		phys_addr_t mmio_phys = cq_mmio_phys_addr(hw, cq_idx);
+#ifdef CONFIG_ARM64
+		if (static_branch_unlikely(&avoid_writecombine))
+			cq->cq_mmio = ioremap(mmio_phys, PAGE_SIZE);
+		else
+#endif
+		cq->cq_mmio = ioremap_wc(mmio_phys, PAGE_SIZE);
+		if (!cq->cq_mmio) {
+			cxidev_warn_once(cdev, "ioremap_wc failed\n");
+			rc = -EFAULT;
+			goto cq_release;
+		}
+	}
+
+	/* Initialize client interface structure */
+	cxi_cq_init(&cq->cass_cq, cq->cmds, cmds_count,
+		    (__force void *)cq->cq_mmio, cq_idx);
+
+	cq_n = cxi_cq_get_cqn(&cq->cass_cq);
+	if (opts->flags & CXI_CQ_IS_TX)
+		rc = setup_hw_tx_cq(hw, cq_n, opts, lni_priv, cq, eq, cp_priv);
+	else
+		rc = setup_hw_tgt_cq(hw, cq_n, opts, lni_priv, cq, eq);
+
+	if (rc)
+		goto cq_unmap;
+
+	if (cq->flags & CXI_CQ_IS_TX)
+		atomic_inc(&hw->stats.txq);
+	else
+		atomic_inc(&hw->stats.tgq);
+
+	cq_debugfs_create(cq_idx, cq, hw, lni_priv);
+
+	spin_lock(&lni_priv->res_lock);
+	list_add_tail(&cq->list, &lni_priv->cq_list);
+	spin_unlock(&lni_priv->res_lock);
+
+	refcount_inc(&lni_priv->refcount);
+	if (eq)
+		refcount_inc(&eq->refcount);
+
+	/* Grant access to the resource from the VF */
+	if (lni_priv->is_vf)
+		cass_config_sriovt(hw, true, lni_priv->vf_num,
+				   (cq->flags & CXI_CQ_IS_TX) ? cq_n :
+								(1024 + cq_n));
+
+	return &cq->cass_cq;
+
+cq_unmap:
+	if (!is_user)
+		iounmap(cq->cq_mmio);
+cq_release:
+	dma_unmap_page(&hw->cdev.pdev->dev, cq->cmds_dma_addr,
+		       cq->cmds_len, DMA_BIDIRECTIONAL);
+
+	if (cq->md_priv) {
+		cass_device_put_pages(cq->md_priv);
+		kfree(cq->md_priv);
+	}
+
+	if (cq->pages) {
+		unpin_user_pages(cq->pages, cq->cmds_len >> PAGE_SHIFT);
+		kfree(cq->pages);
+	}
+
+free_pages:
+	if (cq->cmds_pages)
+		__free_pages(cq->cmds_pages, cmds_order);
+put_id:
+	put_cq_id(cq);
+
+	return ERR_PTR(rc);
+}
+EXPORT_SYMBOL(cxi_cq_alloc_buf_internal);
+
+/**
+ * cxi_cq_alloc_buf() - Allocate a new command queue
+ *
+ * The new CQ is attached to the LNI.
+ * The CQ buffer may be supplied by the user. It must be physically
+ * contiguous.
+ *
+ * @lni: LNI to associate with the command queue.
+ * @evtq: Event Queue to associate with the command queue, for error
+ *        reporting. May be NULL.
+ * @opts_b: Options affecting the CQ.
+ * @numa_node: NUMA node ID CQ memory should be allocated from.
+ *
+ * @return: the command queue or an error pointer
+ */
+struct cxi_cq *cxi_cq_alloc_buf(struct cxi_lni *lni, struct cxi_eq *evtq,
+				const struct cxi_cq_alloc_opts_buf *opts_b,
+				int numa_node)
+{
+	struct cxi_dev *cdev = container_of(lni, struct cxi_lni_priv, lni)->dev;
+
+	if (!cdev->is_physfn)
+		return cxi_cq_alloc_buf_vf(lni, evtq, opts_b, numa_node);
+
+	return cxi_cq_alloc_buf_internal(lni, evtq, opts_b, numa_node, 0, 0);
+}
+EXPORT_SYMBOL(cxi_cq_alloc_buf);
+
+/**
+ * cxi_cq_alloc() - Allocate a new command queue
+ *
+ * The new CQ is attached to attached to the NI.
+ *
+ * @lni: LNI to associate with the command queue.
+ * @evtq: Event Queue to associate with the command queue, for error
+ *        reporting. May be NULL.
+ * @opts: Options affecting the CQ.
+ * @numa_node: NUMA node ID CQ memory should be allocated from.
+ *
+ * @return: the command queue or an error pointer
+ */
+struct cxi_cq *cxi_cq_alloc(struct cxi_lni *lni, struct cxi_eq *evtq,
+			    const struct cxi_cq_alloc_opts *opts,
+			    int numa_node)
+{
+	struct cxi_cq_alloc_opts_buf opts_b = {};
+
+	opts_b.opts = *opts;
+	return cxi_cq_alloc_buf(lni, evtq, &opts_b, numa_node);
+}
+EXPORT_SYMBOL(cxi_cq_alloc);
+
+/* Test whether a transmit CQ is done working */
+static bool is_txq_done(struct cass_dev *hw, int id)
+{
+	union c_cq_txq_rdptr_table rd_ptr;
+	union c_cq_txq_preptr_table pre_ptr;
+	union c_cq_txq_ack_ctr ack_ctr;
+
+	cass_read(hw, C_CQ_TXQ_ACK_CTR(id), &ack_ctr, sizeof(ack_ctr));
+	if (ack_ctr.mem_q_ack_ctr != 0)
+		return false;
+
+	cass_read(hw, C_CQ_TXQ_RDPTR_TABLE(id), &rd_ptr, sizeof(rd_ptr));
+	cass_read(hw, C_CQ_TXQ_PREPTR_TABLE(id), &pre_ptr, sizeof(pre_ptr));
+
+	if (rd_ptr.mem_q_rd_ptr != pre_ptr.mem_q_pre_ptr)
+		return false;
+
+	return true;
+}
+
+/* Test whether a target CQ is done working */
+static bool is_tgq_done(struct cass_dev *hw, int id)
+{
+	union c_cq_tgq_rdptr_table rd_ptr;
+	union c_cq_tgq_preptr_table pre_ptr;
+	union c_cq_tgq_ack_ctr ack_ctr;
+
+	cass_read(hw, C_CQ_TGQ_ACK_CTR(id), &ack_ctr, sizeof(ack_ctr));
+	if (ack_ctr.mem_q_ack_ctr != 0)
+		return false;
+
+	cass_read(hw, C_CQ_TGQ_RDPTR_TABLE(id), &rd_ptr, sizeof(rd_ptr));
+	cass_read(hw, C_CQ_TGQ_PREPTR_TABLE(id), &pre_ptr, sizeof(pre_ptr));
+	if (rd_ptr.mem_q_rd_ptr != pre_ptr.mem_q_pre_ptr)
+		return false;
+
+	return true;
+}
+
+/**
+ * cxi_cq_free() - Destroy a command queue
+ *
+ * @cmdq: the command queue to release
+ */
+void cxi_cq_free(struct cxi_cq *cmdq)
+{
+	struct cxi_cq_priv *cq = container_of(cmdq, struct cxi_cq_priv, cass_cq);
+	struct cxi_lni_priv *lni_priv = cq->lni_priv;
+	struct cxi_eq_priv *eq = cq->eq;
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	unsigned int cq_n = cxi_cq_get_cqn(cmdq);
+	unsigned int mem_q_rd_ptr;
+
+	if (!cdev->is_physfn) {
+		cxi_cq_free_vf(cmdq);
+		return;
+	}
+
+	/* Remove access to this resource from a VF */
+	if (lni_priv->is_vf)
+		cass_config_sriovt(hw, false, lni_priv->vf_num,
+				   (cq->flags & CXI_CQ_IS_TX) ? cq_n :
+								(1024 + cq_n));
+
+	/* CQ invalidation requires disabling and setting the drain bit for the
+	 * CQ. This is done in cass_tx_cq_disable() and cass_tgt_cq_disable().
+	 *
+	 * By the time the read pointer is stable, the buffer is not
+	 * accessed anymore and can be freed.
+	 */
+	if (cq->flags & CXI_CQ_IS_TX) {
+		union c_cq_txq_rdptr_table rd_ptr;
+
+		cass_tx_cq_disable(hw, cq_n);
+
+		cass_read(hw, C_CQ_TXQ_RDPTR_TABLE(cq_n),
+			  &rd_ptr, sizeof(rd_ptr));
+		mem_q_rd_ptr = rd_ptr.mem_q_rd_ptr;
+
+		while (true) {
+			cass_read(hw, C_CQ_TXQ_RDPTR_TABLE(cq_n),
+				  &rd_ptr, sizeof(rd_ptr));
+			if (mem_q_rd_ptr == rd_ptr.mem_q_rd_ptr)
+				break;
+			mem_q_rd_ptr = rd_ptr.mem_q_rd_ptr;
+			udelay(1);
+		}
+	} else {
+		union c_cq_tgq_rdptr_table rd_ptr;
+
+		cass_tgt_cq_disable(hw, cq_n);
+
+		cass_read(hw, C_CQ_TGQ_RDPTR_TABLE(cq_n),
+			  &rd_ptr, sizeof(rd_ptr));
+		mem_q_rd_ptr = rd_ptr.mem_q_rd_ptr;
+		while (true) {
+			cass_read(hw, C_CQ_TGQ_RDPTR_TABLE(cq_n),
+				  &rd_ptr, sizeof(rd_ptr));
+			if (mem_q_rd_ptr == rd_ptr.mem_q_rd_ptr)
+				break;
+			mem_q_rd_ptr = rd_ptr.mem_q_rd_ptr;
+			udelay(1);
+		}
+	}
+
+	if (!lni_priv->is_vf) {
+		if (!(cq->flags & CXI_CQ_USER))
+			iounmap(cq->cq_mmio);
+
+		dma_unmap_page(&hw->cdev.pdev->dev, cq->cmds_dma_addr,
+			       cq->cmds_len, DMA_BIDIRECTIONAL);
+
+		if (cq->cmds_pages) {
+			__free_pages(cq->cmds_pages, cq->cmds_order);
+		} else {
+			if (cq->md_priv) {
+				cass_device_put_pages(cq->md_priv);
+				kfree(cq->md_priv);
+			}
+
+			if (cq->pages) {
+				unpin_user_pages(cq->pages, cq->cmds_len >> PAGE_SHIFT);
+				kfree(cq->pages);
+			}
+		}
+	}
+
+	if (eq)
+		refcount_dec(&eq->refcount);
+
+	spin_lock(&lni_priv->res_lock);
+	list_del(&cq->list);
+	list_add_tail(&cq->list, &lni_priv->cq_cleanups_list);
+	spin_unlock(&lni_priv->res_lock);
+}
+EXPORT_SYMBOL(cxi_cq_free);
+
+/* Go through the list of CQ waiting for cleanup. If the CQ is quiet,
+ * it can be freed, otherwise it stays.
+ *
+ * If force is true, the CQ will be release unconditionally.
+ */
+void finalize_cq_cleanups(struct cxi_lni_priv *lni, bool force)
+{
+	struct cxi_dev *cdev = lni->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	struct cxi_cq_priv *cq;
+	struct cxi_cq_priv *tmp;
+	struct list_head free_list;
+
+	INIT_LIST_HEAD(&free_list);
+
+	spin_lock(&lni->res_lock);
+
+	list_for_each_entry_safe(cq, tmp, &lni->cq_cleanups_list, list) {
+		unsigned int cq_n = cxi_cq_get_cqn(&cq->cass_cq);
+
+		if (!force) {
+			/* Ensure that in-flight commands are done. */
+			if (cq->flags & CXI_CQ_IS_TX) {
+				if (!is_txq_done(hw, cq_n))
+					continue;
+			} else {
+				if (!is_tgq_done(hw, cq_n))
+					continue;
+			}
+		}
+
+		list_del(&cq->list);
+		list_add(&cq->list, &free_list);
+	}
+
+	spin_unlock(&lni->res_lock);
+
+	while ((cq = list_first_entry_or_null(&free_list,
+					      struct cxi_cq_priv, list))) {
+		list_del(&cq->list);
+
+		debugfs_remove(cq->lni_dir);
+		debugfs_remove_recursive(cq->debug_dir);
+
+		refcount_dec(&lni->refcount);
+
+		if (cq->flags & CXI_CQ_IS_TX)
+			atomic_dec(&hw->stats.txq);
+		else
+			atomic_dec(&hw->stats.tgq);
+
+		put_cq_id(cq);
+	}
+}
+
+/**
+ * cxi_cq_user_info() - Get the information to mmap a cq to userspace
+ *
+ * @cmdq: the command queue
+ * @cmds_size: on return, will hold the size of the queue in 64-bytes blocks
+ * @cmds_pages: on return, will hold the pages to mmap
+ * @wp_addr: on return, will hold the CSR to control the CQ
+ * @wp_addr_size: on return, size of the CSR
+ *
+ * @return: 0, or -EINVAL is the CQ isn't for userspace
+ */
+int cxi_cq_user_info(struct cxi_cq *cmdq, size_t *cmds_size,
+		     struct page **cmds_pages, phys_addr_t *wp_addr,
+		     size_t *wp_addr_size)
+{
+	struct cxi_cq_priv *cq = container_of(cmdq, struct cxi_cq_priv, cass_cq);
+	struct cxi_lni_priv *lni_priv = cq->lni_priv;
+	struct cxi_dev *cdev = lni_priv->dev;
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+
+	if (!(cq->flags & CXI_CQ_USER))
+		return -EINVAL;
+
+	/* userspace will need to know what pages to mmap. */
+	*cmds_size = cq->cmds_len;
+	*cmds_pages = cq->cmds_pages;
+
+	*wp_addr = cq_mmio_phys_addr(hw, cq->cass_cq.idx);
+	*wp_addr_size = PAGE_SIZE;
+
+	return 0;
+}
+EXPORT_SYMBOL(cxi_cq_user_info);
+
+/**
+ * cxi_cq_ack_counter_vf() - Get the current ack counter for a CQ in VF
+ *
+ * @cmdq: CQ to get current ack counter for.
+ *
+ * Return: Current ack counter value.
+ */
+static unsigned int cxi_cq_ack_counter_vf(struct cxi_cq *cmdq)
+{
+	struct cxi_cq_priv_vf *cq_priv_vf =
+		container_of(cmdq, struct cxi_cq_priv_vf, cass_cq);
+	struct cxi_dev *dev = cq_priv_vf->lni_priv->dev;
+	const struct cxi_cq_ack_counter_cmd cmd = {
+		.op = CXI_OP_CQ_ACK_COUNTER,
+		.cq = cq_priv_vf->cass_cq.idx,
+	};
+	struct cxi_cq_ack_counter_resp resp;
+	size_t resp_len = sizeof(resp);
+	int rc;
+
+	rc = cxi_send_msg_to_pf(dev, &cmd, sizeof(cmd), &resp, &resp_len);
+	if (rc)
+		return rc;
+
+	return resp.ack_counter;
+}
+
+/**
+ * cxi_cq_ack_counter() - Get the current ack counter for a CQ.
+ *
+ * @cq: CQ to get current ack counter for.
+ *
+ * Return: Current ack counter value.
+ */
+unsigned int cxi_cq_ack_counter(struct cxi_cq *cq)
+{
+	struct cxi_cq_priv *cq_priv =
+		container_of(cq, struct cxi_cq_priv, cass_cq);
+	struct cass_dev *hw =
+		container_of(cq_priv->lni_priv->dev, struct cass_dev, cdev);
+	unsigned int cq_n = cxi_cq_get_cqn(cq);
+	union c_cq_txq_ack_ctr txq_ack_ctr;
+	union c_cq_tgq_ack_ctr tgq_ack_ctr;
+	unsigned int ack_ctr;
+
+	if (!hw->cdev.is_physfn)
+		return cxi_cq_ack_counter_vf(cq);
+
+	if (cq_priv->flags & CXI_CQ_IS_TX) {
+		cass_read(hw, C_CQ_TXQ_ACK_CTR(cq_n), &txq_ack_ctr,
+			  sizeof(txq_ack_ctr));
+		ack_ctr = txq_ack_ctr.mem_q_ack_ctr;
+	} else {
+		cass_read(hw, C_CQ_TGQ_ACK_CTR(cq_n), &tgq_ack_ctr,
+			  sizeof(tgq_ack_ctr));
+		ack_ctr = tgq_ack_ctr.mem_q_ack_ctr;
+	}
+	return ack_ctr;
+}
+EXPORT_SYMBOL(cxi_cq_ack_counter);

@@ -0,0 +1,639 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Create and destroy Cassini counting events.
+ * Copyright 2018, 2024-2026 Hewlett Packard Enterprise Development LP
+ */
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/hpe/cxi/cxi.h>
+#include <linux/iopoll.h>
+
+#include "cass_core.h"
+#include "cxi_internal.h"
+#include "cass_ss1_debugfs.h"
+#include "cxi_vf_cmd.h"
+
+/* Initializes (if init is true) or cancel (if init is false) pending
+ * triggered operations on a CT.
+ */
+static int set_hw_state(struct cass_dev *hw, unsigned int ctn,
+			bool init)
+{
+	void __iomem *csr = cass_csr(hw, C_CQ_CFG_INIT_CT_HW_STATE);
+	union c_cq_cfg_init_ct_hw_state ct_hw_state;
+	int rc;
+
+	mutex_lock(&hw->ct_init_lock);
+
+	ct_hw_state.qw = 0;
+	ct_hw_state.ct_handle = ctn;
+	if (init)
+		ct_hw_state.init = 1;
+	else
+		ct_hw_state.cancel = 1;
+	cass_write(hw, C_CQ_CFG_INIT_CT_HW_STATE, &ct_hw_state.qw,
+		   sizeof(ct_hw_state));
+
+	rc = readq_poll_timeout(csr, ct_hw_state.qw,
+				ct_hw_state.pending == 0, 1, 1000000);
+	cxidev_WARN_ONCE(&hw->cdev, rc, "Timeout waiting for CT operation (%d)\n",
+			 init);
+
+	/* Need to zero cancel and init fields so that they can be reused. */
+	memset(&ct_hw_state, 0, sizeof(ct_hw_state));
+	cass_write(hw, C_CQ_CFG_INIT_CT_HW_STATE, &ct_hw_state.qw,
+		   sizeof(ct_hw_state));
+
+	mutex_unlock(&hw->ct_init_lock);
+
+	return rc;
+}
+
+/* Map the wb buffer using DMA address. Pin buffer if from user space. */
+static int ct_wb_map(struct cass_dev *hw, const struct cxi_ct_priv *ct_priv,
+		     struct ct_wb_desc *desc, struct c_ct_writeback *wb)
+{
+	static const struct c_ct_writeback wb_init = {};
+	struct device *device = &hw->cdev.pdev->dev;
+	dma_addr_t dma_addrs;
+	int rc;
+
+	desc->md_priv = NULL;
+	desc->page = NULL;
+
+	if (!ct_priv->is_user) {
+		*wb = wb_init;
+
+		if (is_vmalloc_addr(wb)) {
+			desc->wb_dma_addr = dma_map_page(device,
+							 vmalloc_to_page(wb),
+							 offset_in_page(wb),
+							 sizeof(*wb), DMA_FROM_DEVICE);
+			desc->is_map_page = 1;
+		} else {
+			desc->wb_dma_addr = dma_map_single(device, wb, sizeof(*wb),
+							   DMA_FROM_DEVICE);
+		}
+		if (dma_mapping_error(&hw->cdev.pdev->dev, desc->wb_dma_addr))
+			return -ENOMEM;
+
+		return 0;
+	}
+
+	rc = pin_user_pages_fast((unsigned long)wb, 1,
+				 FOLL_WRITE | FOLL_LONGTERM, &desc->page);
+	if (rc == 1) {
+		desc->wb_dma_addr = dma_map_single(device,
+						   page_address(desc->page) +
+						   offset_in_page(wb),
+						   sizeof(*wb), DMA_FROM_DEVICE);
+		if (dma_mapping_error(device, desc->wb_dma_addr)) {
+			unpin_user_page(desc->page);
+			desc->page = NULL;
+			return -ENOMEM;
+		}
+
+		return 0;
+	}
+
+	desc->md_priv = cass_device_md(ct_priv->lni_priv, (u64)wb, sizeof(*wb),
+				       &dma_addrs);
+	if (IS_ERR(desc->md_priv))
+		return PTR_ERR(desc->md_priv);
+
+	desc->wb_dma_addr = dma_addrs + offset_in_page(wb);
+
+	return 0;
+}
+
+static void ct_wb_unmap(struct cass_dev *hw, const struct cxi_ct_priv *ct_priv,
+			struct ct_wb_desc *desc)
+{
+	if (desc->wb_dma_addr == 0)
+		return;
+
+	/* The presence of md_priv indicates device memory which is unmapped
+	 * when the pages are released.
+	 */
+	if (desc->md_priv == NULL) {
+		if (desc->is_map_page)
+			dma_unmap_page(&hw->cdev.pdev->dev, desc->wb_dma_addr,
+				       sizeof(struct c_ct_writeback),
+				       DMA_FROM_DEVICE);
+		else
+			dma_unmap_single(&hw->cdev.pdev->dev, desc->wb_dma_addr,
+					 sizeof(struct c_ct_writeback),
+					 DMA_FROM_DEVICE);
+	}
+
+	if (!ct_priv->is_user)
+		return;
+
+	if (desc->page) {
+		unpin_user_page(desc->page);
+		desc->page = NULL;
+	} else if (desc->md_priv) {
+		p2p_ops.put_pages(desc->md_priv);
+		kfree(desc->md_priv);
+		desc->md_priv = NULL;
+	}
+}
+
+static struct cxi_ct *cxi_ct_alloc_vf(struct cxi_lni *lni,
+				      struct c_ct_writeback *wb, bool is_user)
+{
+	struct cxi_dev *dev = container_of(lni, struct cxi_lni_priv_vf, lni)->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cxi_ct_priv *ct_priv;
+	int rc;
+	int ctn;
+	struct cxi_ct_alloc_cmd_vf cmd = {
+		.base.op = CXI_OP_CT_ALLOC,
+		.base.lni = lni->id,
+	};
+	struct cxi_ct_alloc_resp resp = {};
+	size_t resp_len = sizeof(resp);
+	struct cxi_ct_free_cmd free_cmd = {
+		.op = CXI_OP_CT_FREE,
+	};
+
+	/* WB address must be 8-byte aligned. */
+	if (!IS_ALIGNED((u64)wb, 8))
+		return ERR_PTR(-EINVAL);
+
+	ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL);
+	if (!ct_priv)
+		return ERR_PTR(-ENOMEM);
+	ct_priv->lni_priv = container_of(lni, struct cxi_lni_priv, lni);
+	ct_priv->is_user = is_user;
+
+	/* A WB address context of zero means that WB is disabled for the CT. */
+	if (wb) {
+		rc = ct_wb_map(hw, ct_priv, &ct_priv->wb_desc, wb);
+		if (rc)
+			goto free_ct_priv;
+	} else {
+		ct_priv->wb_desc.wb_dma_addr = 0;
+	}
+
+	cmd.wb_dma_addr = ct_priv->wb_desc.wb_dma_addr;
+
+	rc = cxi_send_msg_to_pf(dev, &cmd, sizeof(cmd), &resp, &resp_len);
+	if (rc)
+		goto unmap_wb;
+
+	ctn = resp.ctn;
+	ct_priv->ct.ctn = ctn;
+
+	if (!is_user) {
+		phys_addr_t mmio_phys = hw->regs_base + C_MEMORG_CQ_TOU +
+				ctn * C_TOU_LAUNCH_PAGE_SIZE;
+		ct_priv->ct_mmio = ioremap(mmio_phys, PAGE_SIZE);
+		if (!ct_priv->ct_mmio) {
+			cxidev_warn_once(dev, "ioremap failed\n");
+			rc = -ENOMEM;
+			goto free_pf_ct;
+		}
+	}
+
+	return &ct_priv->ct;
+
+free_pf_ct:
+	free_cmd.ctn = ctn;
+	resp_len = 0;
+	cxi_send_msg_to_pf(dev, &free_cmd, sizeof(free_cmd), NULL, &resp_len);
+unmap_wb:
+	if (wb)
+		ct_wb_unmap(hw, ct_priv, &ct_priv->wb_desc);
+free_ct_priv:
+	kfree(ct_priv);
+	return ERR_PTR(rc);
+}
+
+/**
+ * cxi_ct_alloc_internal() - Allocate a counting event (internal implementation
+ * to handle VF requests)
+ *
+ * @lni: LNI counting event is allocated against
+ * @wb: User defined writeback buffer, must be DMA-able.
+ * @is_user: Non-zero value if the counting event is for userspace.
+ * @wb_dma_addr: Pre-mapped DMA address for writeback buffer (for requests from VF)
+ *
+ * Writeback buffer is optional. If @wb is NULL, the wb field in the returned
+ * cxi_ct will also be NULL. @wb must be NULL in requests coming from VFs, the
+ * writeback buffer (if any) is passed using wb_dma_addr, having already been
+ * mapped by the VF.
+ *
+ * Return: Valid pointer on success. Else, negative errno pointer.
+ */
+struct cxi_ct *cxi_ct_alloc_internal(struct cxi_lni *lni,
+				     struct c_ct_writeback *wb, bool is_user,
+				     dma_addr_t wb_dma_addr)
+{
+	struct cxi_lni_priv *lni_priv =
+		container_of(lni, struct cxi_lni_priv, lni);
+	struct cxi_dev *dev = lni_priv->dev;
+	struct cass_dev *hw =
+		container_of(dev, struct cass_dev, cdev);
+	struct cxi_ct_priv *ct_priv;
+	u16 wb_ac;
+	int rc;
+	int ctn;
+
+	/* WB address must be 8-byte aligned. */
+	if (!IS_ALIGNED((u64)wb, 8))
+		return ERR_PTR(-EINVAL);
+
+	ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL);
+	if (!ct_priv)
+		return ERR_PTR(-ENOMEM);
+	ct_priv->lni_priv = lni_priv;
+	ct_priv->is_user = is_user;
+
+	/* Check the associated service to see if this CT can be allocated */
+	rc = cxi_rgroup_alloc_resource(lni_priv->rgroup, CXI_RESOURCE_CT);
+	if (rc)
+		goto free_ct_priv;
+
+	/*
+	 * A WB address context of zero means that WB is disabled for
+	 * the CT.
+	 */
+	if (wb) {
+		rc = ct_wb_map(hw, ct_priv, &ct_priv->wb_desc, wb);
+		if (rc)
+			goto dec_rsrc_use;
+		wb_ac = lni_priv->phys_ac;
+	} else if (wb_dma_addr) {
+		ct_priv->wb_desc.wb_dma_addr = wb_dma_addr;
+		wb_ac = lni_priv->phys_ac;
+	} else {
+		ct_priv->wb_desc.wb_dma_addr = 0;
+		wb_ac = C_AC_NONE;
+	}
+
+	/* Allocate and configure the counting event. */
+	rc = ida_alloc_range(&hw->ct_table, 1, C_NUM_CTS - 1, GFP_KERNEL);
+	if (rc < 0)
+		goto free_md_priv;
+	ctn = rc;
+
+	if (!is_user) {
+		phys_addr_t mmio_phys = hw->regs_base + C_MEMORG_CQ_TOU +
+				ctn * C_TOU_LAUNCH_PAGE_SIZE;
+		ct_priv->ct_mmio = ioremap(mmio_phys, PAGE_SIZE);
+		if (!ct_priv->ct_mmio) {
+			cxidev_warn_once(dev, "ioremap failed\n");
+			rc = -ENOMEM;
+			goto ct_release;
+		}
+	}
+
+	/* Reset the hardware CT. */
+	rc = set_hw_state(hw, ctn, true);
+	if (rc)
+		goto ct_unmap;
+
+	if (lni_priv->is_vf)
+		cass_config_sriovt(hw, true, lni_priv->vf_num, 2048 + ctn);
+
+	/* Finally the CT can be enabled */
+	cxi_ct_init(&ct_priv->ct, wb, ctn, (__force u64 *)ct_priv->ct_mmio);
+	cass_ct_enable(hw, ctn, lni_priv->lni.rgid, ct_priv->wb_desc.wb_dma_addr, wb_ac);
+
+	/* Track the counting event. */
+	spin_lock(&lni_priv->res_lock);
+	list_add_tail(&ct_priv->entry, &lni_priv->ct_list);
+	refcount_inc(&lni_priv->refcount);
+	atomic_inc(&hw->stats.ct);
+	spin_unlock(&lni_priv->res_lock);
+
+	ct_debugfs_setup(ct_priv->ct.ctn, ct_priv, hw, lni_priv);
+
+	return &ct_priv->ct;
+
+ct_unmap:
+	if (!is_user)
+		iounmap(ct_priv->ct_mmio);
+ct_release:
+	ida_free(&hw->ct_table, ctn);
+free_md_priv:
+	if (wb)
+		ct_wb_unmap(hw, ct_priv, &ct_priv->wb_desc);
+dec_rsrc_use:
+	cxi_rgroup_free_resource(lni_priv->rgroup, CXI_RESOURCE_CT);
+free_ct_priv:
+	kfree(ct_priv);
+
+	return ERR_PTR(rc);
+}
+EXPORT_SYMBOL(cxi_ct_alloc_internal);
+
+/**
+ * cxi_ct_alloc() - Allocate a counting event
+ *
+ * @lni: LNI counting event is allocated against
+ * @wb: User defined writeback buffer, must be DMA-able.
+ * @is_user: Non-zero value if the counting event is for userspace.
+ *
+ * Writeback buffer is optional. If @wb is NULL, the wb field in the returned
+ * cxi_ct will also be NULL.
+ *
+ * Return: Valid pointer on success. Else, negative errno pointer.
+ */
+struct cxi_ct *cxi_ct_alloc(struct cxi_lni *lni,
+			    struct c_ct_writeback *wb, bool is_user)
+{
+	struct cxi_dev *dev = container_of(lni, struct cxi_lni_priv, lni)->dev;
+
+	if (dev->is_physfn)
+		return cxi_ct_alloc_internal(lni, wb, is_user, 0);
+	else
+		return cxi_ct_alloc_vf(lni, wb, is_user);
+}
+EXPORT_SYMBOL(cxi_ct_alloc);
+
+/* Wait for credits_in_use.tou_wb_credits to drain to 0. We then know that
+ * no more writebacks are in progress.
+ */
+static int drain_wb_credits(struct cass_dev *hw)
+{
+	ktime_t timeout;
+	int timeout_us = 1000;
+	union c_cq_sts_credits_in_use in_use;
+
+	timeout = ktime_add_us(ktime_get(), timeout_us);
+
+	for (;;) {
+		cass_read(hw, C_CQ_STS_CREDITS_IN_USE, &in_use,
+			  sizeof(in_use));
+
+		if (in_use.tou_wb_credits == 0)
+			return 0;
+
+		if (ktime_compare(ktime_get(), timeout) > 0)
+			return -ETIMEDOUT;
+
+		usleep_range(5, 20);
+	}
+}
+
+static int cxi_ct_wb_update_vf(struct cxi_ct *ct, struct c_ct_writeback *wb)
+{
+	struct cxi_ct_priv *ct_priv = container_of(ct, struct cxi_ct_priv, ct);
+	struct cxi_dev *dev = ct_priv->lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cxi_ct_wb_update_cmd_vf cmd = {
+		.base.op = CXI_OP_CT_WB_UPDATE,
+		.base.ctn = ct->ctn,
+	};
+	size_t resp_len = 0;
+	struct ct_wb_desc new_desc;
+	int rc;
+
+	if (!wb)
+		return -EINVAL;
+
+	rc = ct_wb_map(hw, ct_priv, &new_desc, wb);
+	if (rc)
+		return rc;
+
+	cmd.wb_dma_addr = new_desc.wb_dma_addr;
+
+	rc = cxi_send_msg_to_pf(dev, &cmd, sizeof(cmd), NULL, &resp_len);
+	if (rc)
+		goto unmap_new_desc;
+
+	/* Previous pinned pages need to be cleaned up. */
+	ct_wb_unmap(hw, ct_priv, &ct_priv->wb_desc);
+	ct_priv->wb_desc = new_desc;
+
+	/* finally, save the new wb */
+	ct->wb = wb;
+
+	return 0;
+
+unmap_new_desc:
+	ct_wb_unmap(hw, ct_priv, &new_desc);
+	return rc;
+}
+
+/**
+ * cxi_ct_wb_update_internal() - Update a counting event with a new writeback
+ * address (internal implementation to handle VF requests)
+ *
+ * Flush by setting c_cq_cfg_credit_limits.tou_wb_credits to 0. When
+ * c_cq_sts_credits_in_use.tou_wb_credits is 0, then no writebacks are
+ * in progress. Re-enable the c_cq_cfg_credit_limits.tou_wb_credits.
+ *
+ * @ct: Counting event
+ * @wb: User defined writeback buffer, must be DMA-able.
+ * @wb_dma_addr: Pre-mapped DMA address for writeback buffer (for requests from VF)
+ *
+ * As documented in cxi_ct_alloc_internal(), @wb must be NULL in requests coming
+ * from VFs, the new writeback buffer is passed using @wb_dma_addr, having
+ * already been mapped by the VF.
+ */
+int cxi_ct_wb_update_internal(struct cxi_ct *ct, struct c_ct_writeback *wb,
+			      dma_addr_t wb_dma_addr)
+{
+	struct cxi_ct_priv *ct_priv = container_of(ct, struct cxi_ct_priv, ct);
+	struct cxi_lni_priv *lni_priv = ct_priv->lni_priv;
+	struct cass_dev *hw =
+		container_of(lni_priv->dev, struct cass_dev, cdev);
+	union c_cq_cfg_credit_limits limits = {};
+	union c_cq_cfg_ct_wb_mem_addr mem = {
+		.wb_mem_context = lni_priv->phys_ac,
+	};
+	struct ct_wb_desc new_desc = {};
+	int wb_credits = 0;
+	int rc = 0;
+
+	// TODO: write pycxi test for this fn + driver test
+
+	if (!wb && !wb_dma_addr)
+		return -EINVAL;
+
+	if (!lni_priv->is_vf) {
+		rc = ct_wb_map(hw, ct_priv, &new_desc, wb);
+		if (rc)
+			return rc;
+	} else {
+		new_desc.wb_dma_addr = wb_dma_addr;
+	}
+
+	mutex_lock(&hw->ct_init_lock);
+
+	/* If wb is valid, it may be in use. */
+	if (ct_priv->wb_desc.wb_dma_addr) {
+		/* stop wb writes */
+		cass_read(hw, C_CQ_CFG_CREDIT_LIMITS, &limits, sizeof(limits));
+		wb_credits = limits.tou_wb_credits;
+		limits.tou_wb_credits = 0;
+		cass_write(hw, C_CQ_CFG_CREDIT_LIMITS, &limits, sizeof(limits));
+
+		rc = drain_wb_credits(hw);
+		if (rc)
+			goto set_limit;
+	}
+
+	mem.wb_mem_addr = new_desc.wb_dma_addr >> 3;
+	cass_write(hw, C_CQ_CFG_CT_WB_MEM_ADDR(ct->ctn), &mem, sizeof(mem));
+	cass_flush_pci(hw);
+
+	/* Previous pinned pages need to be cleaned up. */
+	if (!lni_priv->is_vf)
+		ct_wb_unmap(hw, ct_priv, &ct_priv->wb_desc);
+
+set_limit:
+	if (ct_priv->wb_desc.wb_dma_addr) {
+		limits.tou_wb_credits = wb_credits;
+		cass_write(hw, C_CQ_CFG_CREDIT_LIMITS, &limits, sizeof(limits));
+	}
+
+	/* finally, save the new wb */
+	if (!rc) {
+		ct->wb = wb;
+		ct_priv->wb_desc = new_desc;
+	}
+	mutex_unlock(&hw->ct_init_lock);
+
+	if (rc && !lni_priv->is_vf)
+		ct_wb_unmap(hw, ct_priv, &new_desc);
+
+	return rc;
+}
+EXPORT_SYMBOL(cxi_ct_wb_update_internal);
+
+/**
+ * cxi_ct_wb_update() - Update a counting event with a new writeback address.
+ *
+ * Flush by setting c_cq_cfg_credit_limits.tou_wb_credits to 0. When
+ * c_cq_sts_credits_in_use.tou_wb_credits is 0, then no writebacks are
+ * in progress. Re-enable the c_cq_cfg_credit_limits.tou_wb_credits.
+ *
+ * @ct: Counting event
+ * @wb: User defined writeback buffer, must be DMA-able.
+ */
+int cxi_ct_wb_update(struct cxi_ct *ct, struct c_ct_writeback *wb)
+{
+	struct cxi_ct_priv *ct_priv = container_of(ct, struct cxi_ct_priv, ct);
+
+	if (!ct_priv->lni_priv->dev->is_physfn)
+		return cxi_ct_wb_update_vf(ct, wb);
+	else
+		return cxi_ct_wb_update_internal(ct, wb, 0);
+}
+EXPORT_SYMBOL(cxi_ct_wb_update);
+
+static void cxi_ct_free_vf(struct cxi_ct *ct)
+{
+	struct cxi_ct_priv *ct_priv = container_of(ct, struct cxi_ct_priv, ct);
+	struct cxi_dev *dev = ct_priv->lni_priv->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cxi_ct_free_cmd cmd = {
+		.op = CXI_OP_CT_FREE,
+		.ctn = ct->ctn,
+	};
+	size_t resp_len = 0;
+
+	/* Inform PF to free the CT */
+	cxi_send_msg_to_pf(dev, &cmd, sizeof(cmd), NULL, &resp_len);
+
+	if (!ct_priv->is_user)
+		iounmap(ct_priv->ct_mmio);
+
+	ct_wb_unmap(hw, ct_priv, &ct_priv->wb_desc);
+}
+
+/**
+ * cxi_ct_free() - Free a counting event.
+ * @ct: Counting event
+ */
+void cxi_ct_free(struct cxi_ct *ct)
+{
+	struct cxi_ct_priv *ct_priv = container_of(ct, struct cxi_ct_priv, ct);
+	struct cxi_lni_priv *lni_priv = ct_priv->lni_priv;
+	struct cass_dev *hw =
+		container_of(lni_priv->dev, struct cass_dev, cdev);
+
+	if (!hw->cdev.is_physfn) {
+		cxi_ct_free_vf(ct);
+		return;
+	}
+
+	/* Change ownership to the driver. */
+	cass_ct_disable(hw, ct_priv->ct.ctn);
+
+	if (lni_priv->is_vf)
+		cass_config_sriovt(hw, false, lni_priv->vf_num, 2048 + ct_priv->ct.ctn);
+
+	/* Cancel all pending operations. */
+	set_hw_state(hw, ct_priv->ct.ctn, false);
+
+	spin_lock(&lni_priv->res_lock);
+	list_del(&ct_priv->entry);
+	spin_unlock(&lni_priv->res_lock);
+
+	if (!ct_priv->is_user)
+		iounmap(ct_priv->ct_mmio);
+
+	if (!lni_priv->is_vf)
+		ct_wb_unmap(hw, ct_priv, &ct_priv->wb_desc);
+
+	/* Hang the CT on the LNI pending deletion list, to be
+	 * released later
+	 */
+	spin_lock(&lni_priv->res_lock);
+	list_add_tail(&ct_priv->entry, &lni_priv->ct_cleanups_list);
+	spin_unlock(&lni_priv->res_lock);
+}
+EXPORT_SYMBOL(cxi_ct_free);
+
+/* Finish releasing all the CTs on the deletion pending list */
+void finalize_ct_cleanups(struct cxi_lni_priv *lni)
+{
+	struct cxi_dev *dev = lni->dev;
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	struct cxi_ct_priv *ct;
+
+	while ((ct = list_first_entry_or_null(&lni->ct_cleanups_list,
+					      struct cxi_ct_priv, entry))) {
+		list_del(&ct->entry);
+
+		debugfs_remove(ct->lni_dir);
+		debugfs_remove_recursive(ct->debug_dir);
+
+		refcount_dec(&lni->refcount);
+		atomic_dec(&hw->stats.ct);
+		cxi_rgroup_free_resource(lni->rgroup, CXI_RESOURCE_CT);
+		ida_free(&hw->ct_table, ct->ct.ctn);
+		kfree(ct);
+	}
+}
+
+/**
+ * cxi_ct_user_info() - Get userspace mmap info for counting event doorbell.
+ * @ct: Counting event
+ * @doorbell_addr: On success, will contain the address of the doorbell.
+ * @doorbell_size: On success, will contain the size of the doorbell.
+ *
+ * Return: On success, returns zero. On error, returns a negative error number.
+ */
+int cxi_ct_user_info(struct cxi_ct *ct, phys_addr_t *doorbell_addr,
+		     size_t *doorbell_size)
+{
+	struct cxi_ct_priv *ct_priv = container_of(ct, struct cxi_ct_priv, ct);
+	struct cass_dev *hw = container_of(ct_priv->lni_priv->dev,
+					   struct cass_dev, cdev);
+
+	if (!ct_priv->is_user)
+		return -EINVAL;
+
+	*doorbell_addr = hw->regs_base + C_MEMORG_CQ_TOU +
+		(ct->ctn * C_TOU_LAUNCH_PAGE_SIZE);
+	*doorbell_size = PAGE_SIZE;
+
+	return 0;
+}
+EXPORT_SYMBOL(cxi_ct_user_info);