[PATCH 16/23] net: hpe: ss1: add the Cassini core device layer

@@ -0,0 +1,702 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2021-2022,2025 Hewlett Packard Enterprise Development LP */
+
+/* Cassini HSN cable support */
+
+#include <linux/hpe/sbl/sbl.h>
+#include "cass_core.h"
+#include "cass_cable.h"
+
+static int cass_hdsh_get_format(struct cass_dev *hw, u8 *qsfp_format);
+static void cass_hdsh_get_media(struct cass_dev *hw,
+				struct sbl_media_attr *attr, u8 qsfp_format);
+static void cass_hdsh_get_info(struct cass_dev *hw,
+			       struct sbl_media_attr *attr, u8 qsfp_format);
+static void cass_hdsh_get_length(struct cass_dev *hw,
+				 struct sbl_media_attr *attr, u8 qsfp_format);
+static void cass_hdsh_get_vendor(struct cass_dev *hw,
+				 struct sbl_media_attr *attr, u8 qsfp_format);
+static int cass_link_headshell_down_no_media(struct cass_dev *hw, int reason);
+
+/**
+ * cass_is_cable_present() - Check if we have an HSN cable installed
+ *
+ * @hw: the device
+ *
+ * Return: 1 if cable present, else 0
+ */
+int cass_is_cable_present(struct cass_dev *hw)
+{
+	if (!HW_PLATFORM_ASIC(hw))
+		return 1;
+	return (hw->qsfp_eeprom_page_len != 0);
+}
+
+/**
+ * cass_parse_heashell_data() - Read the QSFP EEPROM and parse into attr
+ *
+ * @hw: the device
+ * @attr: pointer to struct to initialize with cable information
+ * @qsfp_format: What spec this cable CSR space is organized according to.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+int cass_parse_heashell_data(struct cass_dev *hw, struct sbl_media_attr *attr,
+			     u8 *qsfp_format)
+{
+	int rc = 0;
+
+	mutex_lock(&hw->qsfp_eeprom_lock);
+
+	if (hw->qsfp_eeprom_page_len <= 0) {
+		cxidev_err(&hw->cdev,
+			   "Failed to parse headshell data. EEPROM data unavailable!\n");
+		rc = -EIO;
+		goto out_unlock;
+	}
+
+	/* Start by determining the cable format */
+	rc = cass_hdsh_get_format(hw, qsfp_format);
+	if (rc)
+		goto out_unlock;
+
+	/* Read out media type information and parse into attr */
+	cass_hdsh_get_media(hw, attr, *qsfp_format);
+
+	/* Read out the supported speed modes into attr */
+	cass_hdsh_get_info(hw, attr, *qsfp_format);
+
+	/* Read out media length information into attr */
+	cass_hdsh_get_length(hw, attr, *qsfp_format);
+
+	/* Read out vendor information and parse into attr */
+	cass_hdsh_get_vendor(hw, attr, *qsfp_format);
+
+ out_unlock:
+	mutex_unlock(&hw->qsfp_eeprom_lock);
+	return rc;
+}
+
+/**
+ * cass_hdsh_get_format() - Read the QSFP EEPROM and parse out the media format
+ *
+ * @hw: the device
+ * @qsfp_format: pointer to spec this cable CSR space is organized according to
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static int cass_hdsh_get_format(struct cass_dev *hw, u8 *qsfp_format)
+{
+	if (hw->qsfp_eeprom_page0[QSFP_IDENTIFIER_OFFSET] >= SFF8024_TYPE_QSFPDD) {
+		// What revision compliance do they claim
+		if (hw->qsfp_eeprom_page0[QSFP_REV_CMPL_OFFSET] < 0x27)
+			/* Very similar to CMIS 2.0, might need to distinguish
+			 *  someday?
+			 */
+			*qsfp_format = QDD_SFF8636;
+		else {
+			*qsfp_format = QDD_CMIS;
+			/* CMIS V3.0-V4.0 are mostly compatible.
+			 * Version-specific differences are handled in the code.
+			 */
+			/* allow CMIS v5.* */
+			if (hw->qsfp_eeprom_page0[QSFP_REV_CMPL_OFFSET] >= 0x60) {
+				cxidev_err(&hw->cdev,
+					   "Failed to parse headshell data. Bad CMIS rev (%d)!\n",
+					   hw->qsfp_eeprom_page0[QSFP_REV_CMPL_OFFSET]);
+				return -EMEDIUMTYPE;
+			}
+		}
+	} else
+		// Treat anything else as SFF-8636 Format
+		*qsfp_format = QDD_SFF8636;
+
+	return 0;
+}
+
+/**
+ * cass_hdsh_get_media() - Read the QSFP EEPROM and parse out the media type
+ *
+ * @hw: the device
+ * @attr: pointer to struct to initialize with cable information
+ * @qsfp_format: What spec this cable CSR space is organized according to.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static void cass_hdsh_get_media(struct cass_dev *hw,
+				struct sbl_media_attr *attr, u8 qsfp_format)
+{
+	u8 qsfp_media;
+
+	if (qsfp_format == QDD_CMIS)
+		qsfp_media = hw->qsfp_eeprom_page0[CMIS_MEDIA_TYPE_OFFSET];
+	else /* QDD_SFF8636 */
+		qsfp_media = hw->qsfp_eeprom_page0[SFF8436_MEDIA_TYPE_OFFSET];
+	if (qsfp_media >= MEDIA_COPPER_UNEQ)
+		attr->media = SBL_LINK_MEDIA_ELECTRICAL;
+	else
+		attr->media = SBL_LINK_MEDIA_OPTICAL;
+}
+
+/**
+ * cass_hdsh_get_info() - Read the QSFP EEPROM and parse out the media info
+ *
+ * @hw: the device
+ * @attr: pointer to struct to initialize with cable information
+ * @qsfp_format: What spec this cable CSR space is organized according to.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static void cass_hdsh_get_info(struct cass_dev *hw,
+			       struct sbl_media_attr *attr, u8 qsfp_format)
+{
+	if (attr->media == SBL_LINK_MEDIA_ELECTRICAL)
+		// For copper cables, we support all link modes
+		attr->info = SBL_MEDIA_INFO_SUPPORTS_BS_200G |
+			SBL_MEDIA_INFO_SUPPORTS_BJ_100G |
+			SBL_MEDIA_INFO_SUPPORTS_CD_100G |
+			SBL_MEDIA_INFO_SUPPORTS_CD_50G;
+	else {
+		if (qsfp_format == QDD_CMIS)
+			/* TODO - CMIS support is not immediately required */
+			return;
+		/* else: QDD_SFF8636 */
+		/* SFF8636 is not supported, so advertise all link mode
+		 *  are supported.
+		 */
+		attr->info = SBL_MEDIA_INFO_SUPPORTS_BS_200G |
+			SBL_MEDIA_INFO_SUPPORTS_BJ_100G |
+			SBL_MEDIA_INFO_SUPPORTS_CD_100G |
+			SBL_MEDIA_INFO_SUPPORTS_CD_50G;
+	}
+}
+
+/**
+ * cass_hdsh_get_length() - Read the QSFP EEPROM and parse out the media length
+ *
+ * @hw: the device
+ * @attr: pointer to struct to initialize with cable information
+ * @qsfp_format: What spec this cable CSR space is organized according to.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static void cass_hdsh_get_length(struct cass_dev *hw,
+				 struct sbl_media_attr *attr, u8 qsfp_format)
+{
+	u8 qsfp_length;
+	int mult, qsfp_length_mm;
+
+	if (qsfp_format == QDD_CMIS) {
+		qsfp_length = hw->qsfp_eeprom_page0[CMIS_CABLE_LEN_OFFSET];
+		mult = (qsfp_length & 0xC0) >> 6;
+		qsfp_length &= 0x3F;
+		switch (mult) {
+		case 0:
+			attr->len = qsfp_length * 10;
+			break;
+		case 1:
+			attr->len = qsfp_length * 100;
+			break;
+		case 2:
+			attr->len = qsfp_length * 1000;
+			break;
+		case 3:
+			attr->len = qsfp_length * 10000;
+			break;
+		default:
+			attr->len = 0;
+			break;
+		}
+	} else { /* QDD_SFF8636 */
+		qsfp_length = hw->qsfp_eeprom_page0[SFF8436_LINK_LEN_OFFSET];
+		attr->len = qsfp_length * 100;
+	}
+
+	/* TODO - sbl_link_len will be converted to len in cm, rather than
+	 *  this enum. Remove this section when that happens
+	 */
+	qsfp_length_mm = attr->len * 10;
+	switch (qsfp_length_mm) {
+	case 300:
+		attr->len = SBL_LINK_LEN_000_300; break;
+	case 400:
+		attr->len = SBL_LINK_LEN_000_400; break;
+	case 750:
+		attr->len = SBL_LINK_LEN_000_750; break;
+	case 800:
+		attr->len = SBL_LINK_LEN_000_800; break;
+	case 1000:
+		attr->len = SBL_LINK_LEN_001_000; break;
+	case 1100:
+		attr->len = SBL_LINK_LEN_001_100; break;
+	case 1150:
+		attr->len = SBL_LINK_LEN_001_150; break;
+	case 1200:
+		attr->len = SBL_LINK_LEN_001_200; break;
+	case 1400:
+		attr->len = SBL_LINK_LEN_001_400; break;
+	case 1420:
+		attr->len = SBL_LINK_LEN_001_420; break;
+	case 1500:
+		attr->len = SBL_LINK_LEN_001_500; break;
+	case 1600:
+		attr->len = SBL_LINK_LEN_001_600; break;
+	case 1640:
+		attr->len = SBL_LINK_LEN_001_640; break;
+	case 1700:
+		attr->len = SBL_LINK_LEN_001_700; break;
+	case 1900:
+		attr->len = SBL_LINK_LEN_001_900; break;
+	case 1910:
+		attr->len = SBL_LINK_LEN_001_910; break;
+	case 2000:
+		attr->len = SBL_LINK_LEN_002_000; break;
+	case 2100:
+		attr->len = SBL_LINK_LEN_002_100; break;
+	case 2130:
+		attr->len = SBL_LINK_LEN_002_130; break;
+	case 2200:
+		attr->len = SBL_LINK_LEN_002_200; break;
+	case 2300:
+		attr->len = SBL_LINK_LEN_002_300; break;
+	case 2390:
+		attr->len = SBL_LINK_LEN_002_390; break;
+	case 2400:
+		attr->len = SBL_LINK_LEN_002_400; break;
+	case 2500:
+		attr->len = SBL_LINK_LEN_002_500; break;
+	case 2600:
+		attr->len = SBL_LINK_LEN_002_600; break;
+	case 2620:
+		attr->len = SBL_LINK_LEN_002_620; break;
+	case 2700:
+		attr->len = SBL_LINK_LEN_002_700; break;
+	case 2800:
+		attr->len = SBL_LINK_LEN_002_800; break;
+	case 2900:
+		attr->len = SBL_LINK_LEN_002_900; break;
+	case 2990:
+		attr->len = SBL_LINK_LEN_002_990; break;
+	case 3000:
+		attr->len = SBL_LINK_LEN_003_000; break;
+	case 4000:
+		attr->len = SBL_LINK_LEN_004_000; break;
+	case 5000:
+		attr->len = SBL_LINK_LEN_005_000; break;
+	case 6000:
+		attr->len = SBL_LINK_LEN_006_000; break;
+	case 7000:
+		attr->len = SBL_LINK_LEN_007_000; break;
+	case 8000:
+		attr->len = SBL_LINK_LEN_008_000; break;
+	case 10000:
+		attr->len = SBL_LINK_LEN_010_000; break;
+	case 14000:
+		attr->len = SBL_LINK_LEN_014_000; break;
+	case 15000:
+		attr->len = SBL_LINK_LEN_015_000; break;
+	case 19000:
+		attr->len = SBL_LINK_LEN_019_000; break;
+	case 25000:
+		attr->len = SBL_LINK_LEN_025_000; break;
+	case 30000:
+		attr->len = SBL_LINK_LEN_030_000; break;
+	case 35000:
+		attr->len = SBL_LINK_LEN_035_000; break;
+	case 50000:
+		attr->len = SBL_LINK_LEN_050_000; break;
+	case 75000:
+		attr->len = SBL_LINK_LEN_075_000; break;
+	case 100000:
+		attr->len = SBL_LINK_LEN_100_000; break;
+	default:
+		attr->len = SBL_LINK_LEN_INVALID;
+		cxidev_warn(&hw->cdev, "No valid enum found for link length %d!",
+			    qsfp_length_mm);
+		break;
+	}
+}
+
+/**
+ * cass_hdsh_get_vendor() - Read the QSFP EEPROM and parse out the media vendor
+ *
+ * @hw: the device
+ * @attr: pointer to struct to initialize with cable information
+ * @qsfp_format: What spec this cable CSR space is organized according to.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static void cass_hdsh_get_vendor(struct cass_dev *hw,
+				 struct sbl_media_attr *attr, u8 qsfp_format)
+{
+	const char *qsfp_vendor;
+
+	if (qsfp_format == QDD_CMIS)
+		qsfp_vendor = &(hw->qsfp_eeprom_page0[CMIS_VENDOR_OFFSET]);
+	else /* QDD_SFF8636 */
+		qsfp_vendor = &(hw->qsfp_eeprom_page0[SFF8436_VENDOR_OFFSET]);
+
+	if (strnstr(qsfp_vendor, QDD_VENDOR_TE, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_TE;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_LEONI, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_LEONI;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_MOLEX, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_MOLEX;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_HISENSE, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_HISENSE;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_DUST_PHOTONICS, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_DUST_PHOTONICS;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_FINISAR, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_FINISAR;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_LUXSHARE, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_LUXSHARE;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_FIT, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_FIT;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_FT, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_FT;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_MELLANOX, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_MELLANOX;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_HITACHI, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_HITACHI;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_HPE, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_HPE;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_CLOUD_LIGHT, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_CLOUD_LIGHT;
+	else if (strnstr(qsfp_vendor, QDD_VENDOR_AMPHENOL, QSFP_VENDOR_MAX_STR_LEN) != NULL)
+		attr->vendor = SBL_LINK_VENDOR_AMPHENOL;
+	else {
+		cxidev_warn(&hw->cdev, "No valid enum found for link Vendor %.*s",
+			    QSFP_VENDOR_MAX_STR_LEN, qsfp_vendor);
+		attr->vendor = SBL_LINK_VENDOR_INVALID;
+	}
+}
+
+/**
+ * cass_headshell_power_up() - Power headshell up to high power mode
+ *
+ * @hw: the device
+ * @qsfp_format: What spec this cable CSR space is organized according to.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+int cass_headshell_power_up(struct cass_dev *hw, u8 qsfp_format)
+{
+	u8 qsfp_extid, data;
+	int rc = 0;
+
+	mutex_lock(&hw->qsfp_eeprom_lock);
+
+	// Full power up of all data paths + high power mode
+	if (qsfp_format == QDD_CMIS) {
+		/* TODO: aapl script will do the power up */
+		rc = 0;
+	} else if (qsfp_format == QDD_SFF8636) {
+		// If not, power up the QSFP
+		data = (SFF8636_PWR_CLASS_567_ENABLE | SFF8636_PWR_OVERRIDE);
+		qsfp_extid = hw->qsfp_eeprom_page0[SFF8436_EXT_IDENTIFIER_OFFSET];
+		// If the module supports Class8+ set that enable bit too
+		if (qsfp_extid & SFF8636_PWR_CLASS_8)
+			data |= SFF8636_PWR_CLASS_8_ENABLE;
+		rc = uc_cmd_qsfp_write(hw, 0, SFF8436_POWER_DATA_OFFSET,
+				       &data, sizeof(data));
+	} else
+		rc = -EINVAL;
+
+	mutex_unlock(&hw->qsfp_eeprom_lock);
+	return rc;
+}
+
+/**
+ * cass_link_headshell_insert() - Headshell insert
+ *
+ * @hw: the device
+ * @attr: pointer to struct containing cable information
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+int cass_link_headshell_insert(struct cass_dev *hw,
+			       struct sbl_media_attr *attr)
+{
+	int err;
+
+	cass_phy_set_state(CASS_PHY_NOLINK, hw);
+	cxidev_dbg(&hw->cdev, "headshell insert: phy state = %d\n", hw->phy.state);
+
+	if (hw->port->hstate == CASS_HEADSHELL_STATUS_PRESENT) {
+		cxidev_err(&hw->cdev, "headshell insert - already present\n");
+		return -EUCLEAN;
+	}
+
+	if (hw->port->config_state & CASS_MEDIA_CONFIGURED) {
+		cxidev_err(&hw->cdev,
+			   "headshell insert - media already configured\n");
+		return -EUCLEAN;
+	}
+
+	err = hw->link_ops->media_config(hw, attr);
+	if (err) {
+		cxidev_err(&hw->cdev, "headshell insert - media config failed [%d]\n",
+			   err);
+		hw->port->config_state &= ~CASS_MEDIA_CONFIGURED;
+		return err;
+	}
+
+	/*
+	 * check any existing link config is compatible with the new media type
+	 *
+	 *    if not unconfigure link so it must be reconfigured again
+	 *
+	 */
+	if (hw->port->config_state & CASS_LINK_CONFIGURED) {
+		switch (hw->port->lattr.bl.config_target) {
+		case SBL_BASE_LINK_CONFIG_PEC:
+			if (attr->media != SBL_LINK_MEDIA_ELECTRICAL) {
+				cxidev_warn(&hw->cdev,
+					    "headshell insert - link attr PEC incompatible with new media");
+				hw->port->config_state &= ~CASS_LINK_CONFIGURED;
+			}
+			break;
+		case SBL_BASE_LINK_CONFIG_AOC:
+			if (attr->media != SBL_LINK_MEDIA_OPTICAL) {
+				cxidev_warn(&hw->cdev,
+					    "headshell insert - link attr AOC incompatible with new media");
+				hw->port->config_state &= ~CASS_LINK_CONFIGURED;
+			}
+			break;
+		default:
+			cxidev_warn(&hw->cdev,
+				    "headshell insert - link attr incompatible with new media");
+			hw->port->config_state &= ~CASS_LINK_CONFIGURED;
+			break;
+		}
+	}
+
+	hw->port->hstate = CASS_HEADSHELL_STATUS_PRESENT;
+	hw->port->config_state |= CASS_MEDIA_CONFIGURED;
+
+	/*
+	 * if the link is supposed to be up try to restart it
+	 */
+	if ((cass_link_get_state(hw) == CASS_LINK_STATUS_DOWN) &&
+	    (hw->port->lmon_dirn == CASS_LMON_DIRECTION_UP)) {
+
+		if ((hw->port->lattr.options &
+		     CASS_LINK_OPT_RESTART_ON_INSERT)) {
+			cxidev_dbg(&hw->cdev, "starting link after insert\n");
+			cass_lmon_request_up(hw);
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * cass_link_headshell_remove() - Headshell remove
+ *
+ *    Headshell must already be present and media configured.
+ *    Stop link if running and unconfigure media
+ *
+ * @hw: the device
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+int cass_link_headshell_remove(struct cass_dev *hw)
+{
+	int err;
+
+	cass_phy_set_state(CASS_PHY_HEADSHELL_REMOVED, hw);
+	cxidev_dbg(&hw->cdev, "headshell remove: phy state = %d\n", hw->phy.state);
+
+	switch (hw->port->hstate) {
+	case CASS_HEADSHELL_STATUS_NOT_PRESENT:
+		cxidev_err(&hw->cdev, "headshell remove - not present\n");
+		return -EUCLEAN;
+
+	case CASS_HEADSHELL_STATUS_PRESENT:
+		if (!(hw->port->config_state & CASS_MEDIA_CONFIGURED)) {
+			cxidev_err(&hw->cdev,
+				   "headshell remove - media not configured\n");
+			return -EUCLEAN;
+		}
+		break;
+
+	case CASS_HEADSHELL_STATUS_ERROR:
+		break;
+
+	default:
+		cxidev_err(&hw->cdev, "headshell remove - unrecognised state\n");
+		return -EUCLEAN;
+	}
+
+	/* stop link coming up */
+	cass_lmon_set_up_pause(hw, true);
+
+	/* setup new state */
+	hw->port->hstate = CASS_HEADSHELL_STATUS_NOT_PRESENT;
+	cass_link_set_led(hw);
+
+	/* ensure link down and no media configured */
+	err = cass_link_headshell_down_no_media(hw,
+						CASS_DOWN_ORIGIN_HEADSHELL_REMOVED);
+	if (err) {
+		cxidev_err(&hw->cdev,
+			   "headshell remove - down_no_media failed [%d]\n",
+			   err);
+		cass_lmon_set_up_pause(hw, false);
+		return err;
+	}
+
+	cass_lmon_set_up_pause(hw, false);
+	return 0;
+}
+
+/**
+ * cass_link_headshell_error() - Headshell error
+ *
+ *    Any start state
+ *    Stop link and unconfigure media
+ *
+ * @hw: the device
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+int cass_link_headshell_error(struct cass_dev *hw)
+{
+	int err;
+
+	cass_phy_set_state(CASS_PHY_HEADSHELL_REMOVED, hw);
+	cxidev_dbg(&hw->cdev, "headshell error: phy state = %d\n", hw->phy.state);
+
+	/* stop link coming up */
+	cass_lmon_set_up_pause(hw, true);
+
+	/* ensure link down and no media configured */
+	err = cass_link_headshell_down_no_media(hw,
+						CASS_DOWN_ORIGIN_HEADSHELL_ERROR);
+	if (err) {
+		cxidev_err(&hw->cdev,
+			   "headshell error - down_no_media failed [%d]\n",
+			   err);
+		cass_lmon_set_up_pause(hw, false);
+		return err;
+	}
+
+	/* setup new state */
+	hw->port->hstate = CASS_HEADSHELL_STATUS_ERROR;
+	cass_link_set_led(hw);
+
+	cass_lmon_set_up_pause(hw, false);
+	return 0;
+}
+
+/**
+ * cass_link_headshell_down_no_media() - take link down and unconfigure cable
+ *
+ * Common headshell remove/error code to stop link and unconfigure media
+ *
+ * @hw: the device
+ * @reason: the reason the link is going down
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static int cass_link_headshell_down_no_media(struct cass_dev *hw, int reason)
+{
+	int err;
+
+	cxidev_dbg(&hw->cdev, "headshell down no media\n");
+
+	/* ensure link down  */
+	spin_lock(&hw->port->lock);
+	switch (hw->port->lstate) {
+	case CASS_LINK_STATUS_DOWN:
+	case CASS_LINK_STATUS_ERROR:
+	case CASS_LINK_STATUS_UNCONFIGURED:
+	case CASS_LINK_STATUS_UNKNOWN:
+		cxidev_dbg(&hw->cdev, "headshell down - already link down (%s)\n",
+			   cass_link_state_str(hw->port->lstate));
+		spin_unlock(&hw->port->lock);
+		break;
+	default:
+		spin_unlock(&hw->port->lock);
+		err = cass_link_async_down_wait(hw, reason);
+		if (err) {
+			cxidev_err(&hw->cdev,
+				   "headshell down - down_wait failed [%d]\n",
+				   err);
+			return err;
+		}
+		break;
+	}
+
+	/* remove any media configuration */
+	if (hw->port->config_state & CASS_MEDIA_CONFIGURED) {
+		err = hw->link_ops->media_unconfig(hw);
+		if (err) {
+			cxidev_err(&hw->cdev, "headshell down - media unconfig failed [%d]\n", err);
+			return err;
+		}
+		hw->port->config_state &= ~CASS_MEDIA_CONFIGURED;
+	}
+
+	return 0;
+}
+
+/*
+ * Direct media configuration
+ *
+ *   Called by hms to set media for backplane media
+ *
+ *   If you are playing with the backplane (i.e. changing compute blades)
+ *   then leave it to the user to restart the link
+ */
+int cass_link_media_config(struct cass_dev *hw, struct sbl_media_attr *attr)
+{
+	int err;
+
+	cxidev_dbg(&hw->cdev, "media config\n");
+
+	if (hw->port->hstate != CASS_HEADSHELL_STATUS_NOT_PRESENT) {
+		cxidev_dbg(&hw->cdev, "media config - bad state %s\n",
+			   cass_link_headshell_state_str(hw->port->hstate));
+		return -EUCLEAN;
+	}
+
+	if (hw->port->config_state & CASS_MEDIA_CONFIGURED) {
+		cxidev_dbg(&hw->cdev, "media config - media already configured\n");
+		return -EUCLEAN;
+	}
+
+	err = hw->link_ops->media_config(hw, attr);
+	if (err) {
+		cxidev_dbg(&hw->cdev, "media config - media config failed [%d]\n",
+			   err);
+		hw->port->config_state &= ~CASS_MEDIA_CONFIGURED;
+		return err;
+	}
+
+	hw->port->hstate = CASS_HEADSHELL_STATUS_NO_DEVICE;
+	hw->port->config_state |= CASS_MEDIA_CONFIGURED;
+
+	return 0;
+}
+
+/**
+ * cass_link_headshell_state_str() - Get a string for the headshell state
+ *
+ * Return a text string describing a headshell status
+ *
+ * @state: input enum of target headshell state
+ *
+ * Return: This function will always return a text string
+ */
+const char *cass_link_headshell_state_str(enum cass_headshell_status state)
+{
+	switch (state) {
+	case CASS_HEADSHELL_STATUS_UNKNOWN:     return "unknown";
+	case CASS_HEADSHELL_STATUS_PRESENT:     return "present";
+	case CASS_HEADSHELL_STATUS_NOT_PRESENT: return "not present";
+	case CASS_HEADSHELL_STATUS_NO_DEVICE:   return "no device";
+	case CASS_HEADSHELL_STATUS_ERROR:       return "error";
+	default:                                return "unrecognised";
+	}
+}

@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2021 Hewlett Packard Enterprise Development LP */
+#ifndef _CASS_CABLE_H
+#define _CASS_CABLE_H
+
+#define QSFP_IDENTIFIER_OFFSET                   0
+#define QSFP_REV_CMPL_OFFSET                     1
+#define QSFP_VENDOR_MAX_STR_LEN                  16
+
+#define QDD_VENDOR_TE                   "TE Connectivity"
+#define QDD_VENDOR_HISENSE              "Hisense"
+#define QDD_VENDOR_FIT                  "FIT HON TENG"
+#define QDD_VENDOR_FT                   "FT HON TENG"
+#define QDD_VENDOR_MELLANOX             "Mellanox"
+#define QDD_VENDOR_MOLEX                "Molex"
+#define QDD_VENDOR_LEONI                "LEONI"
+#define QDD_VENDOR_HITACHI              "Hitachi Metals"
+#define QDD_VENDOR_LUXSHARE             "LUXSHARE-ICT"
+#define QDD_VENDOR_DUST_PHOTONICS       "DustPhotonics"
+#define QDD_VENDOR_FINISAR              "FINISAR CORP"
+#define QDD_VENDOR_HPE                  "HPE"
+#define QDD_VENDOR_CLOUD_LIGHT          "Cloud Light"
+#define QDD_VENDOR_AMPHENOL             "Amphenol"
+
+// SFF-8024 Specification for SFF Cross Reference -----------------------------
+#define SFF8436_MEDIA_TYPE_OFFSET                147
+#define SFF8436_LINK_LEN_OFFSET                  146
+#define SFF8436_VENDOR_OFFSET                    148
+#define SFF8436_EXT_IDENTIFIER_OFFSET            129
+#define SFF8436_POWER_DATA_OFFSET                93
+#define SFF8436_LOW_POWER_MASK                   0x02
+
+// SFF-8024 TABLE 4-1 IDENTIFIER VALUES
+#define SFF8024_TYPE_UNKNOWN                     0x00
+#define SFF8024_TYPE_GBIC                        0x01
+#define SFF8024_TYPE_MOTHERBOARD                 0x02
+#define SFF8024_TYPE_SFP                         0x03
+#define SFF8024_TYPE_300P_XBI                    0x04
+#define SFF8024_TYPE_XENPAK                      0x05
+#define SFF8024_TYPE_XFP                         0x06
+#define SFF8024_TYPE_XFF                         0x07
+#define SFF8024_TYPE_XFP_E                       0x08
+#define SFF8024_TYPE_XPAK                        0x09
+#define SFF8024_TYPE_X2                          0x0A
+#define SFF8024_TYPE_DWDM_SFP                    0x0B
+#define SFF8024_TYPE_QSFP_INF8438                0x0C
+#define SFF8024_TYPE_QSFP_PLUS                   0x0D
+#define SFF8024_TYPE_CXP                         0x0E
+#define SFF8024_TYPE_SMMHD4X                     0x0F
+#define SFF8024_TYPE_SMMHD8X                     0x10
+#define SFF8024_TYPE_QSFP28                      0x11
+#define SFF8024_TYPE_CXP2                        0x12
+#define SFF8024_TYPE_CDFP                        0x13
+#define SFF8024_TYPE_SMMHD4X_FANOUT              0x14
+#define SFF8024_TYPE_SMMHD8X_FANOUT              0x15
+#define SFF8024_TYPE_CDFP_3                      0x16
+#define SFF8024_TYPE_uQSFP                       0x17
+#define SFF8024_TYPE_QSFPDD                      0x18
+#define SFF8024_TYPE_OSFP8X                      0x19
+#define SFF8024_TYPE_SFPDD                       0x1A
+#define SFF8024_TYPE_DSFP                        0x1B
+#define SFF8024_TYPE_MINILINK4X                  0x1C
+#define SFF8024_TYPE_MINILINK8X                  0x1D
+#define SFF8024_TYPE_QSFP_PLUS_CMIS              0x1E
+
+// SFF-8636 Byte @ 93
+#define SFF8636_SW_RESET                         0x80
+#define SFF8636_PWR_CLASS_8_ENABLE               0x08
+#define SFF8636_PWR_CLASS_567_ENABLE             0x04
+#define SFF8636_PWR_CLASS_LOW                    0x02
+#define SFF8636_PWR_OVERRIDE                     0x01
+
+// SFF-8636 Extended Identifier Byte @ Page 00h byte 129
+#define SFF8636_PWR_CLASS_1234_MASK              0xC0
+#define SFF8636_PWR_CLASS_1                      0x00
+#define SFF8636_PWR_CLASS_2                      0x40
+#define SFF8636_PWR_CLASS_3                      0x80
+#define SFF8636_PWR_CLASS_4_AND_UP               0xC0
+#define SFF8636_PWR_CLASS_8                      0x20
+#define SFF8636_CLEI_IN_PG02                     0x10
+#define SFF8636_CDR_IN_TX                        0x08
+#define SFF8636_CDR_IN_RX                        0x04
+#define SFF8636_PWR_CLASS_567_MASK               0x03
+#define SFF8636_SEE_PWR_CLASS_1234               0x00
+#define SFF8636_PWR_CLASS_5                      0x01
+#define SFF8636_PWR_CLASS_6                      0x02
+#define SFF8636_PWR_CLASS_7                      0x03
+
+// Common Management Interface Specification (CMIS) ---------------------------
+#define CMIS_MEDIA_TYPE_OFFSET                   212
+#define CMIS_CABLE_LEN_OFFSET                    202
+#define CMIS_VENDOR_OFFSET                       129
+
+// CMIS and SFF-8636 Media/Transmitter Technology Types
+#define MEDIA_850_NM_VCSEL                       0x00
+#define MEDIA_1310_NM_VCSEL                      0x01
+#define MEDIA_1550_NM_VCSEL                      0x02
+#define MEDIA_1310_NM_FP                         0x03
+#define MEDIA_1310_NM_DFB                        0x04
+#define MEDIA_1550_NM_DFB                        0x05
+#define MEDIA_1310_NM_EML                        0x06
+#define MEDIA_1550_NM_EML                        0x07
+#define MEDIA_OTHERS                             0x08
+#define MEDIA_1490_NM_DFB                        0x09
+#define MEDIA_COPPER_UNEQ                        0x0A
+#define MEDIA_COPPER_PASSIVE_EQ                  0x0B
+#define MEDIA_COPPER_NEAR_FAR_LMT_ACT_EQ         0x0C
+#define MEDIA_COPPER_FAR_LMT_ACT_EQ              0x0D
+#define MEDIA_COPPER_NEAR_LMT_ACT_EQ             0x0E
+#define MEDIA_COPPER_LIN_ACT_EQ                  0x0F
+
+/**
+ * enum qsfp_format - cable spec types
+ *
+ * @QDD_FMT_UNKNOWN:       Format unknown
+ * @QDD_SFF8636:           Format compatible with SFF-8636
+ * @QDD_CMIS:              Format compatible with CMIS
+ *
+ * QSFP / QSFP-DD Protocol/Specification Types
+ */
+enum qsfp_format {
+	QDD_FMT_UNKNOWN = 0,
+	QDD_SFF8636,
+	QDD_CMIS,
+};
+
+/**
+ * enum cass_headshell_status - cable status
+ *
+ * @CASS_HEADSHELL_STATUS_UNKNOWN:       State unknown
+ * @CASS_HEADSHELL_STATUS_PRESENT:       A headshell is inserted
+ * @CASS_HEADSHELL_STATUS_NOT_PRESENT:   A headshell is not inserted
+ * @CASS_HEADSHELL_STATUS_ERROR:         A headshell has error
+ * @CASS_HEADSHELL_STATUS_NO_DEVICE:     No headshell hardware device
+ *
+ * The status of the media headshell
+ */
+enum cass_headshell_status {
+	CASS_HEADSHELL_STATUS_UNKNOWN          = 0,
+	CASS_HEADSHELL_STATUS_PRESENT          = 1 << 0,
+	CASS_HEADSHELL_STATUS_NOT_PRESENT      = 1 << 1,
+	CASS_HEADSHELL_STATUS_ERROR            = 1 << 2,
+	CASS_HEADSHELL_STATUS_NO_DEVICE        = 1 << 3,
+};
+
+int cass_is_cable_present(struct cass_dev *hw);
+int cass_headshell_power_up(struct cass_dev *hw, u8 qsfp_format);
+int cass_parse_heashell_data(struct cass_dev *hw, struct sbl_media_attr *attr,
+			     u8 *qsfp_format);
+int cass_link_headshell_insert(struct cass_dev *hw,
+			       struct sbl_media_attr *mattr);
+int cass_link_headshell_remove(struct cass_dev *hw);
+int cass_link_headshell_error(struct cass_dev *hw);
+int cass_link_media_config(struct cass_dev *hw, struct sbl_media_attr *attr);
+const char *cass_link_headshell_state_str(enum cass_headshell_status state);
+
+#endif	/* _CASS_CABLE_H */

@@ -0,0 +1,1564 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2020-2026 Hewlett Packard Enterprise Development LP */
+
+/* Cassini device handler */
+
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+#include <linux/iopoll.h>
+#include <linux/seq_file.h>
+#include <linux/phy/phy.h>
+#include <linux/if_vlan.h>
+#include <linux/aer.h>
+#include <linux/hpe/cxi/driver.h>
+
+#include "cass_core.h"
+#include "cass_sbl.h"
+#include <linux/hpe/sbl/sbl.h>
+#include "cass_ss1_debugfs.h"
+#include "cxi_aux_dev.h"
+
+/* Range of available number of IRQs, depending on the number of VFs
+ * configured in Cassini.
+ */
+#define MIN_NUM_IRQS 63
+#define MAX_NUM_IRQS 2048
+
+static unsigned int sct_pid_mask = 0x7;
+module_param(sct_pid_mask, uint, 0444);
+MODULE_PARM_DESC(sct_pid_mask, "Source Connection PID Mask");
+
+static unsigned int hw_rev_override;
+module_param(hw_rev_override, uint, 0644);
+MODULE_PARM_DESC(hw_rev_override, "Hardware revision override");
+
+static bool enable_fgfc = true;
+module_param(enable_fgfc, bool, 0444);
+MODULE_PARM_DESC(enable_fgfc, "Enable/disable fine grain flow control");
+
+#define OUTSTANDING_LIMIT_MAX 2047U
+#define OUTSTANDING_LIMIT_MIN 1U
+#define OUTSTANDING_LIMIT_DEFAULT 64U
+
+static unsigned int get_pkt_limit;
+module_param(get_pkt_limit, uint, 0444);
+MODULE_PARM_DESC(get_pkt_limit,
+		 "Maximum number of outstanding GET packets (1-2047) per MCU");
+
+static unsigned int put_pkt_limit;
+module_param(put_pkt_limit, uint, 0444);
+MODULE_PARM_DESC(put_pkt_limit,
+		 "Maximum number of outstanding PUT packets (1-2047) per MCU");
+
+static unsigned int ioi_ord_limit;
+module_param(ioi_ord_limit, uint, 0444);
+MODULE_PARM_DESC(ioi_ord_limit,
+		 "Maximum number of outstanding ordered packets (1-2047) per MCU");
+
+static unsigned int ioi_unord_limit;
+module_param(ioi_unord_limit, uint, 0444);
+MODULE_PARM_DESC(ioi_unord_limit,
+		 "Maximum number of outstanding unordered packets (1-2047) per MCU");
+
+static bool ioi_enable = true;
+module_param(ioi_enable, bool, 0444);
+MODULE_PARM_DESC(ioi_enable,
+		 "Enable the In-Out-In protocol for puts.");
+
+bool disable_on_error = true;
+module_param(disable_on_error, bool, 0644);
+MODULE_PARM_DESC(disable_on_error,
+		 "Disable cxi device on certain unrecoverable errors");
+
+#ifndef PCI_EXT_CAP_ID_DVSEC
+#define PCI_EXT_CAP_ID_DVSEC 0x23
+#define PCI_DVSEC_HEADER1       0x4 /* Designated Vendor-Specific Header1 */
+#endif
+#ifndef PCI_DVSEC_HEADER1_VID
+#define PCI_DVSEC_HEADER1_VID(x)   ((x) & 0xffff)
+#define PCI_DVSEC_HEADER1_REV(x)   (((x) >> 16) & 0xf)
+#define PCI_DVSEC_HEADER1_LEN(x)   (((x) >> 20) & 0xfff)
+#endif
+
+static void cass_dev_release(struct device *dev)
+{
+	struct cass_dev *hw = container_of(dev, struct cass_dev, class_dev);
+
+	kfree(hw);
+}
+
+/**
+ * cxi_get_csrs_range() - Return the whole physical range of the CSRs
+ *
+ * The BAR0 contains amongst other things the CSRs. At least the retry
+ * handler needs an access to them. The CXI user module needs to mmap
+ * it.
+ *
+ * @cdev: the device
+ * @base: base address
+ * @len: length of the CSRs
+ */
+void cxi_get_csrs_range(struct cxi_dev *cdev, phys_addr_t *base, size_t *len)
+{
+	struct cass_dev *hw = container_of(cdev, struct
+					   cass_dev, cdev);
+
+	*base = hw->regs_base + C_MEMORG_CSR;
+	*len = C_MEMORG_CSR_SIZE;
+
+}
+EXPORT_SYMBOL(cxi_get_csrs_range);
+
+void traffic_shaping_cfg(struct cass_dev *hw)
+{
+	static const union c_oxe_cfg_arb_config arb_config = {
+		.mfs_out_sel = 1,
+		.fill_rate = 9,
+	};
+	const union c_oxe_cfg_arb_head_out_c head_out_c = {
+		.enable = 1,
+		.fill_qty = hw->tsc_fill_qty,
+		.limit = DIV_ROUND_UP(8 * ETHERNET_MAX_FRAME_SIZE, 1024),
+	};
+	const union c_oxe_cfg_arb_head_in_c head_in_c = {
+		.enable = 1,
+		.fill_qty = hw->tsc_fill_qty,
+		.limit = DIV_ROUND_UP(8 * ETHERNET_MAX_FRAME_SIZE, 1024),
+	};
+
+	/* 16 traffic shaping credits are returned at 100MHz frequency 200 Gbps
+	 * performance.
+	 */
+	cass_write(hw, C_OXE_CFG_ARB_CONFIG, &arb_config, sizeof(arb_config));
+	cass_write(hw, C_OXE_CFG_ARB_HEAD_OUT_C, &head_out_c,
+		   sizeof(head_out_c));
+	cass_write(hw, C_OXE_CFG_ARB_HEAD_IN_C, &head_in_c, sizeof(head_in_c));
+}
+
+/* Set the number of packets inflight based on link speed.
+ */
+static unsigned int calculate_packets_inflight(struct cass_dev *hw)
+{
+	unsigned int value;
+	struct cxi_link_info link_info;
+	bool is_link_up;
+
+	is_link_up = cass_sbl_is_link_up(hw);
+	if (!is_link_up) {
+		pr_err("cxi link info error: link is not up yet.\n");
+		return 0;
+	}
+	cxi_link_mode_get(&hw->cdev, &link_info);
+
+	if (link_info.speed == SPEED_400000)
+		value = CASS2_MAX_PACKETS_INFLIGHT;
+	else
+		value = CASS1_MAX_PACKETS_INFLIGHT;
+
+	return value;
+}
+
+/* Set outstanding limits based on kernel module parameters.
+ * If no input, use the calculated packets_inflight.
+ */
+void cass_set_outstanding_limit(struct cass_dev *hw)
+{
+	int i;
+	unsigned int packets_inflight;
+	union c_oxe_cfg_outstanding_limit outstanding_limit;
+
+	packets_inflight = calculate_packets_inflight(hw);
+
+	if (packets_inflight < OUTSTANDING_LIMIT_MIN ||
+	    packets_inflight > OUTSTANDING_LIMIT_MAX) {
+		packets_inflight = OUTSTANDING_LIMIT_DEFAULT;
+	}
+
+	if (get_pkt_limit == 0)
+		get_pkt_limit = packets_inflight;
+	if (get_pkt_limit < OUTSTANDING_LIMIT_MIN ||
+	    get_pkt_limit > OUTSTANDING_LIMIT_MAX) {
+		pr_err("Invalid get_pkt_limit %u. Setting to %u\n",
+		       get_pkt_limit, OUTSTANDING_LIMIT_DEFAULT);
+		get_pkt_limit = OUTSTANDING_LIMIT_DEFAULT;
+	}
+
+	if (put_pkt_limit == 0)
+		put_pkt_limit = packets_inflight;
+	if (put_pkt_limit < OUTSTANDING_LIMIT_MIN ||
+	    put_pkt_limit > OUTSTANDING_LIMIT_MAX) {
+		pr_err("Invalid put_pkt_limit %u. Setting to %u\n",
+		       put_pkt_limit, OUTSTANDING_LIMIT_DEFAULT);
+		put_pkt_limit = OUTSTANDING_LIMIT_DEFAULT;
+	}
+
+	if (ioi_ord_limit == 0)
+		ioi_ord_limit = packets_inflight;
+	if (ioi_ord_limit < OUTSTANDING_LIMIT_MIN ||
+	    ioi_ord_limit > OUTSTANDING_LIMIT_MAX) {
+		pr_err("Invalid ioi_ord_limit %u. Setting to %u\n",
+		       ioi_ord_limit, OUTSTANDING_LIMIT_DEFAULT);
+		ioi_ord_limit = OUTSTANDING_LIMIT_DEFAULT;
+	}
+
+	if (ioi_unord_limit == 0)
+		ioi_unord_limit = packets_inflight;
+	if (ioi_unord_limit < OUTSTANDING_LIMIT_MIN ||
+	    ioi_unord_limit > OUTSTANDING_LIMIT_MAX) {
+		pr_err("Invalid ioi_unord_limit %u. Setting to %u\n",
+		       ioi_unord_limit, OUTSTANDING_LIMIT_DEFAULT);
+		ioi_unord_limit = OUTSTANDING_LIMIT_DEFAULT;
+	}
+
+	for (i = 0; i < C_OXE_CFG_OUTSTANDING_LIMIT_ENTRIES; i++) {
+		cass_read(hw, C_OXE_CFG_OUTSTANDING_LIMIT(i),
+			  &outstanding_limit, sizeof(outstanding_limit));
+		outstanding_limit.get_limit = get_pkt_limit;
+		outstanding_limit.put_limit = put_pkt_limit;
+		outstanding_limit.ioi_ord_limit = ioi_ord_limit;
+		outstanding_limit.ioi_unord_limit = ioi_unord_limit;
+		cass_write(hw, C_OXE_CFG_OUTSTANDING_LIMIT(i),
+			   &outstanding_limit, sizeof(outstanding_limit));
+	}
+}
+
+/* Initialize OXE registers. */
+static void cass_oxe_init(struct cass_dev *hw)
+{
+	int i;
+	union c_oxe_cfg_common cfg_common;
+	union c_oxe_cfg_fgfc fgfc;
+	static const union c_oxe_cfg_arb_drr_out wdrr_out = {
+		.quanta_val0 = DIV_ROUND_UP(PORTALS_MAX_FRAME_SIZE, 32),
+		.quanta_val1 = DIV_ROUND_UP(ETHERNET_MAX_FRAME_SIZE, 32),
+	};
+	static const union c_oxe_cfg_arb_drr_in wdrr_in = {
+		.quanta_val0 = DIV_ROUND_UP(PORTALS_MAX_FRAME_SIZE, 32),
+	};
+	static const union c_oxe_cfg_arb_mfs_out mfs_out = {
+		.mfs0 = DIV_ROUND_UP(PORTALS_MAX_FRAME_SIZE, 32),
+		.mfs1 = DIV_ROUND_UP(ETHERNET_MAX_FRAME_SIZE, 32),
+	};
+	static const union c_oxe_cfg_arb_mfs_in mfs_in = {
+		.mfs0 = DIV_ROUND_UP(PORTALS_MAX_FRAME_SIZE, 32),
+	};
+	static const union c_oxe_cfg_mcu_priority_limit mcu_priority_limit = {
+		.occ_climit = 8,
+		.occ_blimit = 4,
+	};
+	static const union c_oxe_cfg_fgfc_cnt fgfc_cnt = {
+		.vni_mask = 0xffff,
+	};
+	union c_oxe_cfg_fab_param fab_param;
+
+	/* Note: All MCUs must use quanta and mfs value zero. */
+	cass_write(hw, C_OXE_CFG_ARB_DRR_OUT, &wdrr_out, sizeof(wdrr_out));
+	cass_write(hw, C_OXE_CFG_ARB_DRR_IN, &wdrr_in, sizeof(wdrr_in));
+	cass_write(hw, C_OXE_CFG_ARB_MFS_OUT, &mfs_out, sizeof(mfs_out));
+	cass_write(hw, C_OXE_CFG_ARB_MFS_IN, &mfs_in, sizeof(mfs_in));
+
+	traffic_shaping_cfg(hw);
+
+	/* Clear all default reserved buffer class values. */
+	cass_clear(hw, C_OXE_CFG_PCT_BC_CDT(0), C_OXE_CFG_PCT_BC_CDT_SIZE);
+	cass_clear(hw, C_OXE_CFG_BUF_BC_PARAM(0), C_OXE_CFG_BUF_BC_PARAM_SIZE);
+
+	/* Lower MCU arbitration priority limits to prevent a single MCU
+	 * starving other MCUs within a traffic class.
+	 */
+	for (i = 0; i < C_OXE_CFG_MCU_PRIORITY_LIMIT_ENTRIES; i++)
+		cass_write(hw, C_OXE_CFG_MCU_PRIORITY_LIMIT(i),
+			   &mcu_priority_limit, sizeof(mcu_priority_limit));
+
+	for (i = 0; i < C_OXE_CFG_FGFC_CNT_ENTRIES; i++)
+		cass_write(hw, C_OXE_CFG_FGFC_CNT(i), &fgfc_cnt,
+			   sizeof(fgfc_cnt));
+
+	cass_read(hw, C_OXE_CFG_FAB_PARAM, &fab_param, sizeof(fab_param));
+	fab_param.req_32b_enabled = 0;
+	fab_param.rsp_32b_enabled = 0;
+	cass_write(hw, C_OXE_CFG_FAB_PARAM, &fab_param, sizeof(fab_param));
+
+	/* Disable In-Out-In for C1 and Mix system */
+	if (system_type_identifier == CASSINI_1_ONLY ||
+	    system_type_identifier == CASSINI_MIX) {
+		ioi_enable = false;
+	}
+	cass_read(hw, C_OXE_CFG_COMMON, &cfg_common,
+		  sizeof(cfg_common));
+	cfg_common.ioi_enable = ioi_enable;
+	cass_write(hw, C_OXE_CFG_COMMON, &cfg_common,
+		   sizeof(cfg_common));
+
+	cass_read(hw, C_OXE_CFG_FGFC, &fgfc, sizeof(fgfc));
+	fgfc.enable = enable_fgfc;
+	cass_write(hw, C_OXE_CFG_FGFC, &fgfc, sizeof(fgfc));
+}
+
+#define DFA_NID_MASK 0xfffff000U
+
+/* TODO: Allow multiple request and response PCP configurations. Currently only
+ * request PCP 0 and response PCP 1 configuration is used.
+ */
+static void cass_pct_init(struct cass_dev *hw)
+{
+	union c_pct_cfg_ixe_rsp_fifo_limits rsp_fifo_limits;
+	union c_pct_cfg_tgt_clr_req_fifo_limit req_fifo_limit;
+	u32 pid_mask = sct_pid_mask;
+	union c_cq_cfg_ocu_hash_mask hash_mask;
+	union c_pct_cfg_sct_tag_dfa_mask dfa_mask;
+	union c_pct_cfg_timing timing;
+	int i;
+
+	/* Errata 2848 */
+	cass_read(hw, C_PCT_CFG_TGT_CLR_REQ_FIFO_LIMIT,
+		  &req_fifo_limit, sizeof(req_fifo_limit));
+	req_fifo_limit.mst_dealloc_fifo_limit = 7;
+	cass_write(hw, C_PCT_CFG_TGT_CLR_REQ_FIFO_LIMIT,
+		   &req_fifo_limit, sizeof(req_fifo_limit));
+
+	cass_read(hw, C_PCT_CFG_IXE_RSP_FIFO_LIMITS,
+		  &rsp_fifo_limits, sizeof(rsp_fifo_limits));
+	rsp_fifo_limits.smt_cdt_ack_limit = 7;
+	rsp_fifo_limits.spt_cdt_ack_limit = 7;
+	rsp_fifo_limits.sct_cdt_ack_limit = 7;
+	cass_write(hw, C_PCT_CFG_IXE_RSP_FIFO_LIMITS,
+		   &rsp_fifo_limits, sizeof(rsp_fifo_limits));
+
+	/* Set default PCT timeouts */
+	cass_read(hw, C_PCT_CFG_TIMING, &timing, sizeof(timing));
+	timing.spt_timeout_epoch_sel = C1_DEFAULT_SPT_TIMEOUT_EPOCH;
+	timing.sct_idle_epoch_sel = C1_DEFAULT_SCT_IDLE_EPOCH;
+	timing.sct_close_epoch_sel = C1_DEFAULT_SCT_CLOSE_EPOCH;
+	timing.tct_timeout_epoch_sel = C1_DEFAULT_TCT_TIMEOUT_EPOCH;
+	cass_write(hw, C_PCT_CFG_TIMING, &timing, sizeof(timing));
+
+	/* Set SCT PID Mask. CQ and PCT DFA masks must be consistent. */
+	cass_read(hw, C_CQ_CFG_OCU_HASH_MASK, &hash_mask, sizeof(hash_mask));
+
+	/* Mask off PID granule space as a part of SCT mask selection. */
+	pid_mask &= BIT(pid_bits) - 1;
+
+	/* Software uses the top bit of the PID granule space to differentiate
+	 * between unrestricted puts and gets. To support having unrestricted
+	 * puts and gets on different SCTs, the top bit of the PID granule
+	 * space must always be included in SCT selection. This bit is the top
+	 * bit in the endpoint define.
+	 */
+	pid_mask |= BIT(C_DFA_ENDPOINT_DEFINED_BITS - 1);
+
+	hash_mask.dfa = DFA_NID_MASK | pid_mask;
+	cass_write(hw, C_CQ_CFG_OCU_HASH_MASK, &hash_mask, sizeof(hash_mask));
+
+	for (i = 0; i < 4; i++)
+		dfa_mask.dscp[i].dfa_mask = pid_mask;
+
+	for (i = 0; i < C_PCT_CFG_SCT_TAG_DFA_MASK_ENTRIES; i++) {
+		cass_write(hw, C_PCT_CFG_SCT_TAG_DFA_MASK(i), &dfa_mask,
+			   sizeof(dfa_mask));
+
+		if (cass_version(hw, CASSINI_2))
+			cass_write(hw, C2_LPE_CFG_GET_FQ_DFA_MASK(i),
+				   &dfa_mask, sizeof(dfa_mask));
+	}
+
+	/* Clear the default target resource settings. This is configured later
+	 * as a part of traffic class configuration.
+	 */
+	if (cass_version(hw, CASSINI_1)) {
+		cass_clear(hw, C1_PCT_CFG_TRS_TC_DED_LIMIT(0),
+			   C1_PCT_CFG_TRS_TC_DED_LIMIT_SIZE);
+		cass_clear(hw, C1_PCT_CFG_TRS_TC_MAX_LIMIT(0),
+			   C1_PCT_CFG_TRS_TC_MAX_LIMIT_SIZE);
+	} else {
+		cass_clear(hw, C2_PCT_CFG_TRS_TC_DED_LIMIT(0),
+			   C2_PCT_CFG_TRS_TC_DED_LIMIT_SIZE);
+		cass_clear(hw, C2_PCT_CFG_TRS_TC_MAX_LIMIT(0),
+			   C2_PCT_CFG_TRS_TC_MAX_LIMIT_SIZE);
+	}
+	cass_clear(hw, C_PCT_CFG_MST_TC_DED_LIMIT(0),
+		   C_PCT_CFG_MST_TC_DED_LIMIT_SIZE);
+	cass_clear(hw, C_PCT_CFG_MST_TC_MAX_LIMIT(0),
+		   C_PCT_CFG_MST_TC_MAX_LIMIT_SIZE);
+	cass_clear(hw, C_PCT_CFG_TCT_TC_DED_LIMIT(0),
+		   C_PCT_CFG_TCT_TC_DED_LIMIT_SIZE);
+	cass_clear(hw, C_PCT_CFG_TCT_TC_MAX_LIMIT(0),
+		   C_PCT_CFG_TCT_TC_MAX_LIMIT_SIZE);
+};
+
+/* Wait for the hardware to be ready after a reset. */
+static int wait_hw_ready(struct cass_dev *hw)
+{
+	void __iomem *csr = cass_csr(hw, C_RMU_STS_INIT_DONE);
+	union {
+		u64 value;
+		union c_atu_sts_init_done atu_sts_init_done;
+		union c_cq_sts_init_done cq_sts_init_done;
+		union c_lpe_sts_init_done lpe_sts_init_done;
+		union c_oxe_sts_common oxe_sts_common;
+		union c_rmu_sts_init_done rmu_sts_init_done;
+		union c_ee_sts_init_done ee_sts_init_done;
+	} x;
+	int rc;
+
+	csr = cass_csr(hw, C_ATU_STS_INIT_DONE);
+	rc = readq_poll_timeout(csr, x.value,
+				(x.atu_sts_init_done.anchor_init_done == 1 &&
+				 x.atu_sts_init_done.tag_init_done == 1 &&
+				 x.atu_sts_init_done.prb_init_done == 1),
+				1, 1000000);
+	if (rc)
+		return rc;
+
+	csr = cass_csr(hw, C_CQ_STS_INIT_DONE);
+	rc = readq_poll_timeout(csr, x.value,
+				(x.cq_sts_init_done.tou_init_done == 1 &&
+				 x.cq_sts_init_done.cq_init_done == 1),
+				1, 1000000);
+	if (rc)
+		return rc;
+
+	csr = cass_csr(hw, C_LPE_STS_INIT_DONE);
+	rc = readq_poll_timeout(csr, x.value,
+				(x.lpe_sts_init_done.free_mask_init_done == 0xf &&
+				 x.lpe_sts_init_done.ptlte_init_done == 1),
+				1, 1000000);
+	if (rc)
+		return rc;
+
+	csr = cass_csr(hw, C_OXE_STS_COMMON);
+	rc = readq_poll_timeout(csr, x.value,
+				(x.oxe_sts_common.init_done == 1),
+				1, 1000000);
+	if (rc)
+		return rc;
+
+	csr = cass_csr(hw, C_RMU_STS_INIT_DONE);
+	rc = readq_poll_timeout(csr, x.value,
+				(x.rmu_sts_init_done.warm_reset == 0 &&
+				 x.rmu_sts_init_done.init_done == 1),
+				1, 1000000);
+	if (rc)
+		return rc;
+
+	csr = cass_csr(hw, C_EE_STS_INIT_DONE);
+	rc = readq_poll_timeout(csr, x.value,
+				(x.ee_sts_init_done.warm_reset == 0 &&
+				 x.ee_sts_init_done.init_done == 1),
+				1, 1000000);
+	if (rc)
+		return rc;
+
+	return 0;
+}
+
+/* Program the device NID. */
+static void program_nid(struct cass_dev *hw, u32 nid)
+{
+	struct cxi_dev *cdev = &hw->cdev;
+	struct cxi_adev *cadev;
+	union c_oxe_cfg_fab_sfa cfg_fab_sfa = {};
+	union c_lpe_cfg_get_ctrl cfg_get_ctrl;
+	union c_ixe_cfg_dfa cfg_dfa = {};
+	union c_pct_cfg_pkt_misc cfg_pkt_misc;
+
+	cdev->prop.nid = nid;
+
+	cass_read(hw, C_PCT_CFG_PKT_MISC, &cfg_pkt_misc, sizeof(cfg_pkt_misc));
+	cfg_pkt_misc.sfa_nid = cdev->prop.nid;
+	cass_write(hw, C_PCT_CFG_PKT_MISC, &cfg_pkt_misc, sizeof(cfg_pkt_misc));
+
+	cfg_fab_sfa.nid = cdev->prop.nid;
+	cass_write(hw, C_OXE_CFG_FAB_SFA, &cfg_fab_sfa, sizeof(cfg_fab_sfa));
+
+	cass_read(hw, C_LPE_CFG_GET_CTRL, &cfg_get_ctrl, sizeof(cfg_get_ctrl));
+	cfg_get_ctrl.dfa_nid = cdev->prop.nid;
+	cass_write(hw, C_LPE_CFG_GET_CTRL, &cfg_get_ctrl, sizeof(cfg_get_ctrl));
+
+	cfg_dfa.dfa_chk_en = 0xFF;
+	cfg_dfa.dfa_nid = cdev->prop.nid;
+	cass_write(hw, C_IXE_CFG_DFA, &cfg_dfa, sizeof(cfg_dfa));
+
+	cxi_send_async_event(cdev, CXI_EVENT_NID_CHANGED);
+
+	/*
+	 * The auxiliary device is freed when it is removed, which clears
+	 * cxi_adev under this lock first. Hold the lock for the whole
+	 * notification so removal waits for it and cannot free the device
+	 * under it.
+	 */
+	mutex_lock(&hw->cxi_adev_lock);
+	cadev = hw->cxi_adev;
+	if (cadev)
+		blocking_notifier_call_chain(&cadev->nid_notifier, nid, cdev);
+	mutex_unlock(&hw->cxi_adev_lock);
+}
+
+void cxi_set_nid(struct cxi_dev *cdev, u32 nid)
+{
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+
+	/* The NID cannot be set from a VF */
+	if (!cdev->is_physfn)
+		return;
+
+	program_nid(hw, nid);
+	hw->cdev.prop.nid_configured = true;
+}
+EXPORT_SYMBOL(cxi_set_nid);
+
+void cxi_set_nid_from_mac(struct cxi_dev *cdev, const u8 *addr)
+{
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+	u32 nid;
+
+	/* The NID cannot be set from a VF */
+	if (!cdev->is_physfn)
+		return;
+
+	/* If the NID has already been explicitly configured, do not override it. */
+	if (hw->cdev.prop.nid_configured)
+		return;
+
+	nid = ether_addr_to_u64(addr) & 0xFFFFFULL;
+	program_nid(hw, nid);
+	cxidev_warn(&hw->cdev,
+		    "NID has not been explicitly set. NID derived from MAC: 0x%x\n", nid);
+}
+EXPORT_SYMBOL(cxi_set_nid_from_mac);
+
+/* Determine the Cassini link speed. When CCIX Extended Speed Mode
+ * (ESM) is enabled, non-standard speeds exist.
+ */
+static int get_pcie_link_speed(struct cass_dev *hw)
+{
+	struct pci_dev *pdev = hw->cdev.pdev;
+	enum pci_bus_speed speed;
+	u32 val;
+	int pos;
+	int rc;
+
+	/* Get the regular PCIe speed first */
+	speed = pcie_get_speed_cap(pdev);
+	switch (speed) {
+	case PCIE_SPEED_2_5GT:
+	case PCIE_SPEED_5_0GT:
+	case PCIE_SPEED_8_0GT:
+	case PCIE_SPEED_16_0GT:
+	case PCIE_SPEED_32_0GT:
+		hw->pcie_link_speed = speed;
+		break;
+
+	default:
+		hw->pcie_link_speed = CASS_SPEED_UNKNOWN;
+		break;
+	}
+
+	/* ESM is only present on first generation Cassini hardware. */
+	if (cass_version(hw, CASSINI_2))
+		return 0;
+
+	pos = 0;
+	while ((pos = pci_find_next_ext_capability(pdev, pos,
+						   PCI_EXT_CAP_ID_DVSEC))) {
+		pci_read_config_dword(pdev, pos + PCI_DVSEC_HEADER1, &val);
+		if (PCI_DVSEC_HEADER1_VID(val) == 0x0001 &&
+		    PCI_DVSEC_HEADER1_REV(val) == 0x1 &&
+		    PCI_DVSEC_HEADER1_LEN(val) == 0x040)
+			break;
+	}
+	if (pos == 0) {
+		/* The header is not present when using PCIe
+		 * passthrough with the vfio-pci driver.
+		 */
+		cxidev_warn(&hw->cdev, "Cannot find the PCIe DVSEC header\n");
+		return 0;
+	}
+
+	/* ESM control register */
+	rc = pci_read_config_dword(pdev, pos + 0x18, &val);
+	if (rc) {
+		cxidev_err(&hw->cdev, "Cannot get ESM status\n");
+		return -EIO;
+	}
+	hw->esm_active = !!(val & BIT(15));
+
+	if (!hw->esm_active)
+		return 0;
+
+	/* ESM PCIe link speed, from the ESM status register */
+	rc = pci_read_config_dword(pdev, pos + 0x14, &val);
+	if (rc) {
+		cxidev_err(&hw->cdev, "Cannot get PCIe link speed\n");
+		return -EIO;
+	}
+
+	switch (val & 0x7f) {
+	case 0x1:
+		hw->pcie_link_speed = PCIE_SPEED_2_5GT;
+		break;
+	case 0x2:
+		hw->pcie_link_speed = PCIE_SPEED_5_0GT;
+		break;
+	case 0x3:
+		hw->pcie_link_speed = PCIE_SPEED_8_0GT;
+		break;
+	case 0x6:
+		hw->pcie_link_speed = PCIE_SPEED_16_0GT;
+		break;
+	case 0xa:
+		hw->pcie_link_speed = CASS_SPEED_20_0GT;
+		break;
+	case 0xf:
+		hw->pcie_link_speed = CASS_SPEED_25_0GT;
+		break;
+	default:
+		hw->pcie_link_speed = CASS_SPEED_UNKNOWN;
+		break;
+	}
+
+	return 0;
+}
+
+static const unsigned int cfg_crnc_csr[] = {
+	C_ATU_CFG_CRNC,
+	C_CQ_CFG_CRNC,
+	C_EE_CFG_CRNC,
+	C_HNI_CFG_CRNC,
+	C_HNI_PML_CFG_CRNC,
+	C_IXE_CFG_CRNC,
+	C_LPE_CFG_CRNC,
+	C_MB_CFG_CRNC,
+	C_MST_CFG_CRNC,
+	C_OXE_CFG_CRNC,
+	C_PARBS_CFG_CRNC,
+	C_PCT_CFG_CRNC,
+	C_PI_CFG_CRNC,
+	C_PI_IPD_CFG_CRNC,
+	C_RMU_CFG_CRNC,
+};
+
+/* Initializes the hardware. */
+static int init_hw(struct cass_dev *hw)
+{
+	int ret;
+	int i;
+	u64 do_dump = 0;
+	union c_lpe_cfg_min_free_shft lpe_cfg_min_free_shift = {
+		.min_free_shft = min_free_shift,
+	};
+	static const union c_mb_cfg_crmc cfg_crmc = {
+		.ring_timeout = 0xff,
+		.rd_addr_invalid_decerr_disable = 1,
+	};
+	static const union c_mb_cfg_misc cfg_mask = {
+		.nic_warm_rst0 = 1,
+		.nic_warm_rst1 = 1,
+		.nic_warm_rst2 = 1,
+		.nic_warm_rst3 = 1,
+		.nic_warm_rst4 = 1,
+		.nic_warm_rst5 = 1,
+		.nic_warm_rst6 = 1,
+		.nic_warm_rst7 = 1,
+		.nic_warm_rst8 = 1,
+		.nic_warm_rst9 = 1,
+	};
+	union c_mb_cfg_misc cfg;
+
+	/* Trigger a warm reset of the hardware - Only the bits common
+	 * to Cassini 1 & 2 are used here.
+	 */
+	cass_read(hw, C_MB_CFG_MISC, &cfg, sizeof(cfg));
+	cfg.qw |= cfg_mask.qw;
+	cass_write(hw, C_MB_CFG_MISC, &cfg, sizeof(cfg));
+
+	/* This CSR must be cleared while warm reset is asserted. */
+	cass_clear(hw, C_EE_DBG_LM_CB_TO_EQ_MAP(0), C_EE_DBG_LM_CB_TO_EQ_MAP_SIZE);
+
+	/* De-assert warm reset */
+	cfg.qw &= ~cfg_mask.qw;
+	cass_write(hw, C_MB_CFG_MISC, &cfg, sizeof(cfg));
+
+	cass_flush_pci(hw);
+
+	if (wait_hw_ready(hw)) {
+		cxidev_err(&hw->cdev, "chip not ready\n");
+		return -ENOTSUPP;
+	}
+
+	cass_init(hw);
+
+	/* Fake hardware initialization */
+	if (HW_PLATFORM_NETSIM(hw))
+		cass_write(hw, NICSIM_CONFIG_DUMP, &do_dump, sizeof(do_dump));
+
+	for (i = 0; i < C_MB_CFG_CRMC_ENTRIES; i++)
+		cass_write(hw, C_MB_CFG_CRMC(i), &cfg_crmc, sizeof(cfg_crmc));
+
+	for (i = 0; i < ARRAY_SIZE(cfg_crnc_csr); i++) {
+		static const union c_atu_cfg_crnc cfg_crnc = {
+			.csr_timeout = 0xfd,
+		};
+
+		/* Use c_atu_cfg_crnc as it's the same structure for
+		 * all CSRs
+		 */
+		cass_write(hw, cfg_crnc_csr[i], &cfg_crnc, sizeof(cfg_crnc));
+	}
+
+	cass_write(hw, C_LPE_CFG_MIN_FREE_SHFT, &lpe_cfg_min_free_shift,
+		   sizeof(lpe_cfg_min_free_shift));
+
+	if (cass_version(hw, CASSINI_2)) {
+		union c2_lpe_cfg_match_sel cfg_match_sel = {
+			.vni_cfg = vni_matching,
+		};
+
+		hw->cdev.prop.vni_matching = vni_matching;
+		cass_write(hw, C2_LPE_CFG_MATCH_SEL, &cfg_match_sel,
+			   sizeof(cfg_match_sel));
+	}
+
+	ret = cass_dmac_init(hw);
+	if (ret)
+		goto __init_hw_err_0;
+
+	ret = cass_telem_init(hw);
+	if (ret)
+		goto __init_hw_err_1;
+
+	cass_pct_init(hw);
+	cass_hni_init(hw);
+	cass_tle_init(hw);
+	cass_cq_init(hw);
+	cass_oxe_init(hw);
+
+	ret = cass_ixe_init(hw);
+	if (ret)
+		goto __init_hw_err_2;
+
+	cass_pte_init(hw);
+	ret = cass_atu_init(hw);
+	if (ret)
+		goto __init_hw_err_2;
+
+	cass_ee_init(hw);
+	ret = cass_tc_init(hw);
+	if (ret)
+		goto __init_hw_err_3;
+
+	cxi_dev_init_eth_tx_profile(hw);
+
+	return 0;
+
+__init_hw_err_3:
+	cass_atu_fini(hw);
+
+__init_hw_err_2:
+	cass_hni_fini(hw);
+	cass_telem_fini(hw);
+
+__init_hw_err_1:
+	cass_dmac_fini(hw);
+
+__init_hw_err_0:
+	return ret;
+}
+
+static void fini_hw(struct cass_dev *hw)
+{
+	cxi_eth_tx_profile_cleanup(hw);
+	cass_tc_fini(hw);
+	cass_atu_fini(hw);
+	cass_ixe_fini(hw);
+	cass_hni_fini(hw);
+	cass_telem_fini(hw);
+	cass_dmac_fini(hw);
+}
+
+/* Check whether the hardware is supported, and set the version. */
+static int is_supported(struct cass_dev *hw)
+{
+	cass_read(hw, C_MB_STS_REV, &hw->rev, sizeof(hw->rev));
+
+	if (!HW_PLATFORM_ASIC(hw) && !HW_PLATFORM_Z1(hw) &&
+	    !HW_PLATFORM_NETSIM(hw)) {
+		cxidev_err(&hw->cdev, "device %x/%x/%x/%x/%x is not supported\n",
+			   hw->rev.vendor_id, hw->rev.device_id,
+			   hw->rev.platform, hw->rev.proto, hw->rev.rev);
+		return -ENOTSUPP;
+	}
+
+	if (hw->rev.vendor_id == PCI_VENDOR_ID_CRAY &&
+	    hw->rev.device_id == PCI_DEVICE_ID_CASSINI_1) {
+		if (hw->rev.rev == 1)
+			hw->cdev.prop.cassini_version = CASSINI_1_0;
+		else if (hw->rev.rev == 2)
+			hw->cdev.prop.cassini_version = CASSINI_1_1;
+		else
+			goto unsupported;
+	} else if (hw->rev.vendor_id == PCI_VENDOR_ID_HPE &&
+		   (hw->rev.device_id == PCI_DEVICE_ID_CASSINI_2_PF ||
+		    hw->rev.device_id == PCI_DEVICE_ID_CASSINI_2_VF)) {
+		if (hw->rev.rev == 1)
+			hw->cdev.prop.cassini_version = CASSINI_2_0;
+		else
+			goto unsupported;
+	}
+
+	if (hw_rev_override) {
+		if (hw_rev_override != CASSINI_1_0 &&
+		    hw_rev_override != CASSINI_1_1 &&
+		    hw_rev_override != CASSINI_2_0)
+			goto unsupported;
+
+		hw->cdev.prop.cassini_version = hw_rev_override;
+	}
+
+	if (cass_version(hw, CASSINI_2) && HW_PLATFORM_Z1(hw) &&
+	    (hw->rev.proto & 0x001) == 1) {
+		cxidev_info(&hw->cdev, "device has reduced PCT tables\n");
+		hw->reduced_pct_tables = true;
+	}
+
+	return 0;
+
+unsupported:
+	cxidev_err(&hw->cdev, "device %x/%x/%x/%x/%x is not supported\n",
+		   hw->rev.vendor_id, hw->rev.device_id,
+		   hw->rev.platform, hw->rev.proto, hw->rev.rev);
+	return -ENOTSUPP;
+}
+
+static int cass_link_fini_once(struct cass_dev *hw, bool *link_started)
+{
+	int rtn;
+
+	if (!*link_started)
+		return 0;
+
+	rtn = hw->link_ops->link_fini(hw);
+	if (!rtn)
+		*link_started = false;
+
+	return rtn;
+}
+
+static int cass_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	int i;
+	int rc = 0;
+	struct cass_dev *hw;
+	bool is_physfn;
+	bool link_started = false;
+	bool class_dev_initialized = false;
+	int pos;
+
+	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
+	if (!hw)
+		return -ENOMEM;
+
+	pci_set_drvdata(pdev, hw);
+	hw->cdev.pdev = pdev;
+	INIT_LIST_HEAD(&hw->lni_list);
+	spin_lock_init(&hw->lni_lock);
+	refcount_set(&hw->refcount, 1);
+
+	atomic_set(&hw->stats.lni, 0);
+
+	/* Initializes the RMU domains */
+	atomic_set(&hw->stats.domain, 0);
+	ida_init(&hw->rmu_index_table);
+	ida_init(&hw->domain_table);
+	spin_lock_init(&hw->domain_lock);
+	spin_lock_init(&hw->pid_pool_lock);
+	spin_lock_init(&hw->rmu_lock);
+
+	/* Initialize the EQs */
+	atomic_set(&hw->stats.eq, 0);
+	ida_init(&hw->eq_index_table);
+	spin_lock_init(&hw->eq_shadow_lock);
+	mutex_init(&hw->init_eq_hw_state);
+
+	/* Initialize the communication profiles. */
+	ida_init(&hw->cp_table);
+	mutex_init(&hw->cp_lock);
+
+	/* Initialize the transmit and target CQs. */
+	atomic_set(&hw->stats.txq, 0);
+	atomic_set(&hw->stats.tgq, 0);
+	ida_init(&hw->cq_table);
+	spin_lock_init(&hw->cq_shadow_lock);
+	mutex_init(&hw->cq_init_lock);
+
+	/* Initialize the PTs */
+	atomic_set(&hw->stats.pt, 0);
+	ida_init(&hw->pte_table);
+	ida_init(&hw->pt_index_table);
+	ida_init(&hw->set_list_table);
+	ida_init(&hw->multicast_table);
+	spin_lock_init(&hw->lpe_shadow_lock);
+	spin_lock_init(&hw->rmu_portal_list_lock);
+	mutex_init(&hw->mst_table_lock);
+	mutex_init(&hw->pte_transition_sm_lock);
+	atomic_set(&hw->plec_count, 0);
+
+	/* Initialize the CTs. */
+	atomic_set(&hw->stats.ct, 0);
+	mutex_init(&hw->ct_init_lock);
+	ida_init(&hw->ct_table);
+
+	ida_init(&hw->lni_table);
+	cass_rgid_init(hw);
+
+	ida_init(&hw->md_index_table);
+
+	for (i = 0; i < C_PE_COUNT; i++)
+		ida_init(&hw->le_pool_ids[i]);
+
+	ida_init(&hw->tle_pool_ids);
+	mutex_init(&hw->msg_relay_lock);
+	mutex_init(&hw->tc_lock);
+	INIT_LIST_HEAD(&hw->tc_list);
+	mutex_init(&hw->cxi_adev_lock);
+	for (i = 0; i < ARRAY_SIZE(hw->vfs); i++)
+		mutex_init(&hw->vfs[i].notif_lock);
+
+	/* Initialize RMU Ethernet resource management */
+	idr_init(&hw->rmu_eth_idr);
+	mutex_init(&hw->rmu_eth_lock);
+
+	mutex_init(&hw->err_flg_mutex);
+	INIT_LIST_HEAD(&hw->err_flg_list);
+	spin_lock_init(&hw->sfs_err_flg_lock);
+
+	spin_lock_init(&hw->mst_match_done_lock);
+	mutex_init(&hw->mst_update_lock);
+
+	spin_lock_init(&hw->sbl_state_lock);
+
+	mutex_init(&hw->qsfp_eeprom_lock);
+	cass_phy_init(hw);
+
+	INIT_DELAYED_WORK(&hw->lni_cleanups_work, lni_cleanups_work);
+	mutex_init(&hw->lni_cleanups_lock);
+	INIT_LIST_HEAD(&hw->lni_cleanups_list);
+
+	mutex_init(&hw->sbus_mutex);
+	mutex_init(&hw->get_ctrl_mutex);
+	mutex_init(&hw->amo_remap_to_pcie_fadd_mutex);
+
+	hw->tsc_fill_qty = TSC_DEFAULT_FILL_QTY;
+	hw->uc_platform = CUC_BOARD_TYPE_UNKNOWN;
+
+	/* Enable Device */
+	rc = pci_enable_device(pdev);
+	if (rc) {
+		cxidev_err(&hw->cdev, "pci_enable_device() failed.\n");
+		goto hw_free;
+	}
+
+	/* The is_physfn and is_virtfn fields of struct pci_dev are only set if
+	 * SR-IOV has been successfully enabled. This means that we cannot rely on
+	 * them to identify physical vs. virtual functions in all cases. A PF on
+	 * real hardware without SR-IOV support, a passthrough PF attached to a VM,
+	 * or a VF attached to a VM, will all have is_physfn = is_virtfn = 0.
+	 *
+	 * C2 (and most other PCI devices) have distinct device IDs for PF and VF.
+	 * However, C1 uses the same ID for both, so we identify PF vs. VF by the
+	 * size of BAR 0.
+	 */
+
+	switch (id->device) {
+	case PCI_DEVICE_ID_CASSINI_1:
+		is_physfn = pci_resource_len(pdev, MMIO_BAR) >= MMIO_BAR_LEN_PF;
+		break;
+	case PCI_DEVICE_ID_CASSINI_2_PF:
+		is_physfn = true;
+		break;
+	case PCI_DEVICE_ID_CASSINI_2_VF:
+		is_physfn = false;
+		break;
+	default:
+		cxidev_err(&hw->cdev, "Unknown device ID: %04x", id->device);
+		return -EOPNOTSUPP;
+	}
+	hw->cdev.is_physfn = is_physfn;
+
+	hw->pci_disabled = false;
+
+	rc = pci_enable_pcie_error_reporting(pdev);
+	if (rc)
+		cxidev_err(&hw->cdev, "PCIe AER not enabled: %d\n", rc);
+
+	/* Set PID space variables for NIC. */
+	hw->cdev.prop.pid_bits = pid_bits;
+	hw->cdev.prop.pid_count = (1 << pid_bits) - 1; /* -1 is wildcard */
+	hw->cdev.prop.pid_granule =
+		(1 << (DFA_EP_BITS - hw->cdev.prop.pid_bits));
+
+	hw->cdev.prop.min_free_shift = min_free_shift;
+
+	/* Set the NID to invalid */
+	hw->cdev.prop.nid = CXI_INVALID_NID;
+
+	/* Enable DMA */
+	pci_set_master(pdev);
+
+	dma_set_max_seg_size(&hw->cdev.pdev->dev, UINT_MAX);
+
+	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(57));
+	if (rc) {
+		cxidev_err(&hw->cdev, "dma_set_mask_and_coherent failed: %d\n", rc);
+		goto dev_disable;
+	}
+
+	/* Enable PCIe AMO requests */
+	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
+				 PCI_EXP_DEVCTL2_ATOMIC_REQ);
+
+	/* Make sure VF 10-bit tag support bit is set for correct counting of
+	 * PI_IPD_pti_tarb_blocked_on_tag.
+	 */
+#define PCI_SRIOV_CTRL_VF_10BIT_TAG 0x20
+	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
+	if (pos) {
+		u16 word;
+
+		pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &word);
+		word |= PCI_SRIOV_CTRL_VF_10BIT_TAG;
+		pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, word);
+	}
+
+	/* Sanity Check. Check BAR 0 is defined and mapped */
+	if (!(pci_resource_flags(pdev, MMIO_BAR) & IORESOURCE_MEM)) {
+		cxidev_err(&hw->cdev, "Incorrect Bar configuration\n");
+		goto dev_disable;
+	}
+
+	/* Take ownership of memory region */
+	rc = pci_request_regions(pdev, "cxi");
+	if (rc) {
+		cxidev_err(&hw->cdev, "pci_request_regions() failed.\n");
+		goto dev_disable;
+	}
+
+	/* Map the base registers */
+	hw->regs_base = pci_resource_start(pdev, MMIO_BAR);
+	if (is_physfn) {
+		hw->regs = ioremap(hw->regs_base + C_MEMORG_CSR,
+				   C_MEMORG_CSR_SIZE);
+		if (!hw->regs) {
+			cxidev_err(&hw->cdev, "ioremap failed\n");
+			goto free_regions;
+		}
+
+		/* Check hardware is supported */
+		rc = is_supported(hw);
+		if (rc)
+			goto unmap_regions;
+
+		/* Check for input and hardware version consistency */
+		if ((system_type_identifier == CASSINI_1_ONLY &&
+		     !cass_version(hw, CASSINI_1)) ||
+		    (system_type_identifier == CASSINI_2_ONLY &&
+		     !cass_version(hw, CASSINI_2))) {
+			cxidev_err(&hw->cdev,
+				   "NIC Version and system_type_identifier (%u) are inconsistent\n",
+				   system_type_identifier);
+			goto unmap_regions;
+		}
+
+		hw->cdev.system_type_identifier = system_type_identifier;
+
+		rc = get_pcie_link_speed(hw);
+		if (rc)
+			goto unmap_regions;
+
+#if defined(CXI_DISABLE_SRIOV)
+		rc = pci_sriov_set_totalvfs(pdev, 0);
+		if (rc)
+			goto unmap_regions;
+#endif
+	}
+
+	hw->with_vf_support = pdev->is_physfn || pci_msix_vec_count(pdev) != 2;
+
+	hw->cdev.cxi_num = atomic_inc_return(&cxi_num);
+	dev_set_name(&hw->class_dev, "cxi%u", hw->cdev.cxi_num);
+	strcpy(hw->cdev.name, dev_name(&hw->class_dev));
+
+	cxi_set_link_ops(hw);
+
+	rc = cass_irq_init(hw);
+	if (rc) {
+		cxidev_err(&hw->cdev, "cass_irq_init failed: %d\n", rc);
+		goto unmap_regions;
+	}
+
+	if (is_physfn) {
+		/* NIC address needs to be assigned before initialization. */
+		rc = init_hw(hw);
+		if (rc) {
+			cxidev_err(&hw->cdev, "init_hw failed: %d\n", rc);
+			goto free_irqs;
+		}
+
+		rc = register_error_handlers(hw);
+		if (rc)
+			goto fini_hw;
+
+		rc = cass_register_uc(hw);
+		if (rc) {
+			cxidev_err(&hw->cdev, "Cannot register the uC: %d\n", rc);
+			goto free_error_handlers;
+		}
+
+		switch (hw->uc_platform) {
+		case CUC_BOARD_TYPE_SAWTOOTH:
+		case CUC_BOARD_TYPE_BRAZOS:
+			if (!cass_version(hw, CASSINI_1)) {
+				cxidev_err(&hw->cdev, "Invalid Cassini 1 platform: %d\n",
+					   hw->uc_platform);
+				goto unregister_uc;
+			}
+			break;
+		case CUC_BOARD_TYPE_WASHINGTON:
+		case CUC_BOARD_TYPE_KENNEBEC:
+		case CUC_BOARD_TYPE_SOUHEGAN:
+			if (!cass_version(hw, CASSINI_2)) {
+				cxidev_err(&hw->cdev, "Invalid Cassini 2 platform: %d\n",
+					   hw->uc_platform);
+				goto unregister_uc;
+			}
+			break;
+		default:
+			cxidev_err(&hw->cdev, "Unknown uC platform %d\n",
+				   hw->uc_platform);
+			goto unregister_uc;
+		}
+
+		if (HW_PLATFORM_ASIC(hw) || HW_PLATFORM_NETSIM(hw)) {
+			/* These platforms have a micro-controller to fetch the
+			 * NIC ID from
+			 */
+			rc = uc_cmd_get_nic_id(hw);
+			if (rc)
+				goto unregister_uc;
+		} else {
+			hw->uc_nic = 0;
+		}
+
+		rc = uc_cmd_get_mac(hw);
+		if (rc)
+			eth_random_addr(hw->default_mac_addr);
+		uc_cmd_set_link_leds(hw, LED_FAST_GRN);
+
+		if (HW_PLATFORM_ASIC(hw)) {
+			/* Program MAC with burned-in MAC. */
+			ether_addr_copy(hw->cdev.mac_addr, hw->default_mac_addr);
+		} else {
+			u64 reg;
+			u8 mac_addr[ETH_ALEN];
+
+			/* Pull MAC from a scratch register on netsim and derive NID from it. */
+			if (HW_PLATFORM_NETSIM(hw))
+				cass_read(hw, NICSIM_NIC_ID, &reg, 8);
+			else
+				cass_read(hw, C_MB_DBG_SCRATCH(1), &reg, 8);
+
+			u64_to_ether_addr(reg, mac_addr);
+			ether_addr_copy(hw->cdev.mac_addr, mac_addr);
+			program_nid(hw, ether_addr_to_u64(hw->cdev.mac_addr) & 0xFFFFFULL);
+		}
+
+		/* SBL init must happen after uC init, since SBL needs to know
+		 * which NIC is associated with this driver instance
+		 */
+		rc = hw->link_ops->link_start(hw);
+		if (rc)
+			goto unregister_uc;
+		link_started = true;
+
+		cass_tc_get_hni_pause_cfg(hw);
+
+		if (cass_version(hw, CASSINI_1)) {
+			rc = cass_register_port(hw);
+			if (rc)
+				goto link_fini;
+		}
+
+		rc = cass_ptp_init(hw);
+		if (rc)
+			goto unregister_port;
+
+		rc = cxi_dmac_desc_set_alloc(&hw->cdev, 1, "pt_id");
+		if (rc < 0)
+			goto cass_ptp_fini;
+		hw->dmac_pt_id = rc;
+
+		hw->mst_entries = dma_alloc_coherent(&pdev->dev,
+						     C_MST_DBG_MST_TABLE_SIZE,
+						     &hw->mst_entries_dma_addr,
+						     GFP_KERNEL);
+		if (!hw->mst_entries)
+			goto cass_dmac_fini;
+
+		rc = cxi_dmac_desc_set_add(&hw->cdev, hw->dmac_pt_id,
+					   hw->mst_entries_dma_addr,
+					   C_MST_DBG_MST_TABLE(0),
+					   C_MST_DBG_MST_TABLE_SIZE);
+		if (rc)
+			goto cass_mst_fini;
+	}
+
+	/* Set after TC/QOS init */
+	hw->cdev.untagged_eth_pcp = hw->qos.untagged_eth_pcp;
+
+	hw->cdev.prop.device_rev = hw->rev.rev;
+	hw->cdev.prop.device_proto = hw->rev.proto;
+	hw->cdev.prop.device_platform = hw->rev.platform;
+
+	hw->max_eth_rxsize = VLAN_ETH_FRAME_LEN;
+
+	cass_probe_debugfs_init(hw);
+
+	/* Hack to grant exclusive access to userspace for QSFP
+	 * firmware flashing. The SL driver must not be able to query
+	 * the QSFP module while an upgrade is happening. Since the UC
+	 * accesses are serialized, we shouldn't need anything
+	 * stronger than that. More ideally, we should have this file
+	 * attached to the i2c debugfs entry, but this was only made
+	 * available in kernel 6.8.
+	 */
+	debugfs_create_bool("i2c_locked", 0644, hw->debug_dir, &hw->uc_i2c_locked);
+
+	/* RX and TX profile setup */
+	cass_dev_rx_tx_profiles_init(hw);
+
+	/* Resource Group setup */
+	cass_dev_rgroup_init(hw);
+
+	/* Service setup */
+	rc = cass_svc_init(hw);
+	if (rc)
+		goto rgroup_fini;
+
+	rc = cxi_configfs_device_init(hw);
+	if (rc) {
+		pr_err("configfs initialize failed\n");
+		goto svc_fini;
+	}
+
+	if (!is_physfn) {
+		struct cxi_get_dev_properties_cmd cmd = {
+			.op = CXI_OP_GET_DEV_PROPERTIES,
+		};
+		struct cxi_properties_info resp;
+		size_t reply_len = sizeof(resp);
+
+		rc = cass_vf_init(hw);
+		if (rc)
+			goto configfs_fini;
+
+		/* Retrieve the device properties */
+		rc = cxi_send_msg_to_pf(&hw->cdev, &cmd, sizeof(cmd),
+					&resp, &reply_len);
+		if (rc != 0) {
+			cxidev_err(&hw->cdev, "BAD: Reply has return code %d\n", rc);
+			goto vf_fini;
+		} else if (reply_len != sizeof(resp)) {
+			cxidev_err(&hw->cdev,
+				   "BAD: Reply has unexpected length %zu\n",
+				   reply_len);
+			goto vf_fini;
+		} else {
+			cxidev_dbg(&hw->cdev, "Reply is valid\n");
+
+			hw->cdev.prop = resp;
+		}
+	}
+
+	/* Export device information. */
+	rc = create_sysfs_properties(hw);
+	if (rc)
+		goto vf_fini;
+
+	/* Register the device with sysfs */
+	hw->class_dev.class = &cxi_class;
+	hw->class_dev.parent = &pdev->dev;
+	hw->class_dev.release = cass_dev_release;
+	dev_set_drvdata(&hw->class_dev, hw);
+	class_dev_initialized = true;
+	if (device_register(&hw->class_dev)) {
+		destroy_sysfs_properties(hw);
+		goto vf_fini;
+	}
+
+	cxi_add_device(&hw->cdev);
+
+	if (is_physfn) {
+		cass_phy_start(hw, true);
+		start_pcie_monitoring(hw);
+	}
+
+	/* RDMA is optional; the NIC and Ethernet work without it. */
+	rc = cxi_ib_add_adev(pdev, hw, hw->cdev.cxi_num);
+	if (rc)
+		cxidev_warn(&hw->cdev,
+			    "RDMA auxiliary device not created: %d\n", rc);
+
+	return 0;
+
+vf_fini:
+	if (!is_physfn)
+		cass_vf_fini(hw);
+
+configfs_fini:
+	cxi_configfs_cleanup(hw);
+svc_fini:
+	cass_svc_fini(hw);
+rgroup_fini:
+	cass_dev_rgroup_fini(hw);
+	cass_dev_rx_tx_profiles_fini(hw);
+	debugfs_remove_recursive(hw->debug_dir);
+cass_mst_fini:
+	if (is_physfn) {
+		rc = cass_link_fini_once(hw, &link_started);
+		if (rc) {
+			WARN(1, "Cassini link teardown failed during probe unwind: %d\n", rc);
+			return rc;
+		}
+	}
+	if (is_physfn)
+		dma_free_coherent(&pdev->dev, C_MST_DBG_MST_TABLE_SIZE,
+				  hw->mst_entries, hw->mst_entries_dma_addr);
+cass_dmac_fini:
+	if (is_physfn) {
+		rc = cass_link_fini_once(hw, &link_started);
+		if (rc) {
+			WARN(1, "Cassini link teardown failed during probe unwind: %d\n", rc);
+			return rc;
+		}
+	}
+	if (is_physfn)
+		cxi_dmac_desc_set_free(&hw->cdev, hw->dmac_pt_id);
+cass_ptp_fini:
+	if (is_physfn) {
+		rc = cass_link_fini_once(hw, &link_started);
+		if (rc) {
+			WARN(1, "Cassini link teardown failed during probe unwind: %d\n", rc);
+			return rc;
+		}
+	}
+	if (is_physfn)
+		cass_ptp_fini(hw);
+unregister_port:
+	if (is_physfn) {
+		rc = cass_link_fini_once(hw, &link_started);
+		if (rc) {
+			WARN(1, "Cassini link teardown failed during probe unwind: %d\n", rc);
+			return rc;
+		}
+	}
+	if (is_physfn)
+		if (cass_version(hw, CASSINI_1))
+			cass_unregister_port(hw);
+link_fini:
+	if (is_physfn) {
+		rc = cass_link_fini_once(hw, &link_started);
+		if (rc) {
+			WARN(1, "Cassini link teardown failed during probe unwind: %d\n", rc);
+			return rc;
+		}
+	}
+unregister_uc:
+	if (is_physfn) {
+		uc_cmd_set_link_leds(hw, LED_OFF);
+		cass_unregister_uc(hw);
+	}
+free_error_handlers:
+	if (is_physfn)
+		deregister_error_handlers(hw);
+fini_hw:
+	if (is_physfn)
+		fini_hw(hw);
+free_irqs:
+	cass_irq_fini(hw);
+unmap_regions:
+	if (is_physfn)
+		iounmap(hw->regs);
+free_regions:
+	pci_release_regions(pdev);
+dev_disable:
+	cxidev_err(&hw->cdev, "deregistering device\n");
+
+	pci_clear_master(pdev);
+	pci_disable_pcie_error_reporting(pdev);
+	pci_disable_device(pdev);
+hw_free:
+	pci_set_drvdata(pdev, NULL);
+	if (class_dev_initialized)
+		put_device(&hw->class_dev);
+	else
+		kfree(hw);
+
+	return -ENODEV;
+}
+
+static void cass_remove(struct pci_dev *pdev)
+{
+	struct cass_dev *hw = pci_get_drvdata(pdev);
+	int rtn;
+
+	if (hw->cxi_adev) {
+		cxi_ib_remove_adev(hw);
+	}
+
+	cxi_remove_device(&hw->cdev);
+	destroy_sysfs_properties(hw);
+	debugfs_lookup_and_remove("port", hw->debug_dir);
+	debugfs_lookup_and_remove("counters", hw->debug_dir);
+
+	if (hw->cdev.is_physfn) {
+		stop_pcie_monitoring(hw);
+		cass_phy_stop(hw, true);
+		rtn = hw->link_ops->link_fini(hw);
+		if (rtn) {
+			WARN(1, "Cassini link teardown failed during remove: %d; aborting device teardown\n",
+			     rtn);
+			return;
+		}
+	}
+
+	cancel_delayed_work_sync(&hw->lni_cleanups_work);
+	lni_cleanups(hw, true);
+
+	/* Destroy RMU Ethernet resource management */
+	idr_destroy(&hw->rmu_eth_idr);
+
+	cass_rgid_fini(hw);
+	cxi_configfs_cleanup(hw);
+	cass_svc_fini(hw);
+	cass_dev_rgroup_fini(hw);
+	cass_dev_rx_tx_profiles_fini(hw);
+
+	debugfs_remove_recursive(hw->debug_dir);
+
+	if (hw->cdev.is_physfn) {
+		if (hw->num_vfs)
+			cass_sriov_configure(hw->cdev.pdev, 0);
+		dma_free_coherent(&pdev->dev, C_MST_DBG_MST_TABLE_SIZE,
+				  hw->mst_entries, hw->mst_entries_dma_addr);
+		cxi_dmac_desc_set_free(&hw->cdev, hw->dmac_pt_id);
+		cass_ptp_fini(hw);
+		uc_cmd_set_link_leds(hw, LED_OFF);
+		cass_unregister_uc(hw);
+		if (cass_version(hw, CASSINI_1))
+			cass_unregister_port(hw);
+		deregister_error_handlers(hw);
+		fini_hw(hw);
+	} else {
+		cass_vf_fini(hw);
+	}
+
+	cass_irq_fini(hw);
+
+	if (hw->cdev.is_physfn)
+		iounmap(hw->regs);
+	pci_release_regions(pdev);
+	pci_clear_master(pdev);
+	pci_disable_pcie_error_reporting(pdev);
+	if (!hw->pci_disabled)
+		pci_disable_device(pdev);
+
+	cxidev_WARN_ONCE(&hw->cdev, !refcount_dec_and_test(&hw->refcount),
+			 "Resource leaks - HW refcount not zero: %d\n",
+			 refcount_read(&hw->refcount));
+	device_unregister(&hw->class_dev);
+}
+
+void cass_disable_device(struct pci_dev *pdev, const char *reason)
+{
+	struct cass_dev *hw = pci_get_drvdata(pdev);
+
+	if (reason)
+		cxidev_err(&hw->cdev, "Disabling device. Reason: %s", reason);
+	else
+		cxidev_err(&hw->cdev, "Disabling device");
+
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+
+	hw->pci_disabled = true;
+}
+
+void uncor_cb(struct cass_dev *hw, unsigned int irq, bool is_ext,
+	      unsigned int bitn)
+{
+	char reason[200];
+
+	if (!disable_on_error)
+		return;
+
+	snprintf(reason, sizeof(reason),
+		 "Unrecoverable error: %s with bitn: %d",
+		 (*hw->err_handlers)[irq][is_ext ? 1 : 0].csr_name_lo, bitn);
+
+	cass_disable_device(hw->cdev.pdev, reason);
+}
+
+static struct pci_device_id cass_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CRAY, PCI_DEVICE_ID_CASSINI_1) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_HPE, PCI_DEVICE_ID_CASSINI_2_PF) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_HPE, PCI_DEVICE_ID_CASSINI_2_VF) },
+	{ 0, },
+};
+
+static struct pci_driver cass_pci_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = cass_ids,
+	.probe = cass_probe,
+	.remove = cass_remove,
+	.sriov_configure = cass_sriov_configure,
+};
+
+MODULE_DEVICE_TABLE(pci, cass_ids);
+
+int hw_register(void)
+{
+	int rc;
+
+	rc = genl_register_family(&cxierr_genl_family);
+	if (rc)
+		return rc;
+
+	rc = pci_register_driver(&cass_pci_driver);
+
+	if (rc)
+		genl_unregister_family(&cxierr_genl_family);
+
+	return rc;
+}
+
+void hw_unregister(void)
+{
+	pci_unregister_driver(&cass_pci_driver);
+	genl_unregister_family(&cxierr_genl_family);
+}
+
+/**
+ * cxi_retry_handler_running() - Check if retry handler is running.
+ *
+ * @cdev: CXI device.
+ *
+ * Return: True if retry handler is running. Else, false.
+ */
+bool cxi_retry_handler_running(struct cxi_dev *cdev)
+{
+	struct cass_dev *hw = container_of(cdev, struct cass_dev, cdev);
+
+	return hw->pct_eq_n != C_EQ_NONE;
+}
+EXPORT_SYMBOL(cxi_retry_handler_running);

@@ -0,0 +1,1472 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2020-2026 Hewlett Packard Enterprise Development LP */
+
+#ifndef _CASS_CORE_H
+#define _CASS_CORE_H
+
+#include <linux/device.h>
+#include <linux/iova.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/rbtree.h>
+#include <linux/workqueue.h>
+#include <net/netlink.h>
+#include <net/genetlink.h>
+#include <linux/ptp_clock_kernel.h>
+#include <linux/etherdevice.h>
+#include <linux/completion.h>
+#include <linux/hrtimer.h>
+#include <linux/hwmon.h>
+#include <linux/socket.h>
+#include <linux/i2c.h>
+
+#include "cxi_link.h"
+#include "cxi_core.h"
+#include "cuc_cxi.h"	/* from firmware_cassini/ */
+#include "cass_sbl.h"
+#include "pldm_cxi.h"
+#include "cass_sl.h"
+#include "cxi_qos_profiles.h"
+#include "cass_rx_tx_profile.h"
+#include "cass_rgroup.h"
+#include "cxi_config.h"
+#include "cass_p2p.h"
+#include "cass_vf.h"
+
+#define PCI_VENDOR_ID_CRAY         0x17db
+#define PCI_DEVICE_ID_CASSINI_1    0x0501
+#define PCI_VENDOR_ID_HPE          0x1590
+#define PCI_DEVICE_ID_CASSINI_2_PF 0x0371
+#define PCI_DEVICE_ID_CASSINI_2_VF 0x0372
+
+#define CXI_MODULE_NAME "cxi_ss1"
+
+/* The following are IPv6 next header values used for IPv6 extension headers.
+ * https://www.iana.org/assignments/ipv6-parameters/ipv6-parameters.xhtml
+ */
+#define IPV6_EH_HOPOPTS 0
+#define IPV6_EH_ROUTE 43
+#define IPV6_EH_FRAG 44
+#define IPV6_EH_ESP 50
+#define IPV6_EH_AH 51
+#define IPV6_EH_OPTS 60
+#define IPV6_EH_MOBILITY 135
+#define IPV6_EH_HIP 139
+#define IPV6_EH_SHIM6 140
+#define IPV6_EH_RSVD1 253
+#define IPV6_EH_RSVD2 254
+
+#define HW_PLATFORM(_hw) ((_hw)->rev.platform)
+#define HW_PLATFORM_ASIC(_hw) (HW_PLATFORM(_hw) == C_PLATFORM_ASIC)
+#define HW_PLATFORM_Z1(_hw) (HW_PLATFORM(_hw) == C_PLATFORM_Z1)
+#define HW_PLATFORM_NETSIM(_hw) (HW_PLATFORM(_hw) == C_PLATFORM_NETSIM)
+
+#define C_MAX_CSR_ERR_INFO 3
+#define C_MAX_CSR_CNTRS   3
+
+/* HPC MTU is fixed at 2k. The largest HPC header is 64 bytes. */
+#define C_MAX_HPC_MTU (2048 + 64)
+
+/* PCI BARs containing the CSRs and CQ/EQ areas. */
+#define MMIO_BAR 0
+
+/* Expected length of PCI BAR 0 for a Physical Function. Used to differentiate
+ * between Virtual Function (512M BAR 0) and Physical Function (2G BAR 0) under
+ * SR-IOV.
+ */
+#define MMIO_BAR_LEN_PF 2147483648
+
+/* Number of error interrupts */
+#define NUM_ERR_INTS 30
+
+#define PORTALS_MAX_FRAME_SIZE 2111U
+#define ETHERNET_MAX_FRAME_SIZE 9022U
+#define TSC_DEFAULT_FILL_QTY 16U
+
+#define PLEC_SIZE 256U
+
+enum cass_phy_state {
+	CASS_PHY_DOWN,
+	CASS_PHY_READY,
+	CASS_PHY_HALTED,
+	CASS_PHY_UP,
+	CASS_PHY_RUNNING,
+	CASS_PHY_NOLINK,
+	CASS_PHY_HEADSHELL_REMOVED,
+};
+
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s:%d " fmt, __func__, __LINE__
+
+#define atu_debug(fmt, ...) \
+do {						\
+	if (more_debug)				\
+		pr_debug(fmt, ##__VA_ARGS__);	\
+} while (0)
+
+/* Helper to get current task network namespace inode id safely. */
+#define CURRENT_NETNS_ID() ({					\
+	unsigned int __netns_id = 0;				\
+	struct task_struct *__ct = current;			\
+	rcu_read_lock();					\
+	if (__ct->nsproxy && __ct->nsproxy->net_ns)		\
+		__netns_id = __ct->nsproxy->net_ns->ns.inum;	\
+	rcu_read_unlock();					\
+	__netns_id;						\
+})
+
+#define ALLOC_IOVA_FAST(iovad, len, end) alloc_iova_fast(iovad, len, end, true)
+#define FREE_IOVA_FAST free_iova_fast
+
+#define INIT_IOVA_DOMAIN(iovad, ps, base, end) \
+		init_iova_domain(iovad, ps, base)
+
+#define ATU_HMM_PFN_TO_PFN(hpfn) page_to_pfn(hmm_pfn_to_page(hpfn))
+
+#define ATU_PFN_INVALID(hpfn) \
+	(!(hpfn & HMM_PFN_VALID) || \
+	 !(hpfn & HMM_PFN_WRITE) || \
+	  (hpfn & HMM_PFN_ERROR))
+#define ATU_PFN_WRITE(hpfn) ((hpfn & HMM_PFN_WRITE))
+#define ATU_PFN_VALID(hpfn) ((hpfn & HMM_PFN_VALID))
+
+/* AER is automatically enabled by the PCI core. */
+static inline int pci_enable_pcie_error_reporting(void *p) { return 0; }
+static inline void pci_disable_pcie_error_reporting(void *p) {}
+
+/* SR-IOV requires VFIO with KVM support enabled and kernel >= 5.19 */
+#if !defined(CONFIG_KVM_VFIO)
+#define CXI_DISABLE_SRIOV
+#endif
+
+#define ATU_FAULT_RETRY_MAX 16
+#define ATU_PTE_PTR (1 << 6)
+#define ATU_PTE_READ (1UL << 57)
+#define ATU_PTE_WRITE (1UL << 58)
+#define ATU_PTE_LEAF (1UL << 59)
+#define ATU_PTE_PRESENT (1UL << 60)
+#define ATU_HMM_RANGE_TIMEOUT 1000
+#define ATU_PTE_MASK (~(ATU_PTE_READ | ATU_PTE_WRITE | ATU_PTE_LEAF | \
+				ATU_PTE_PRESENT))
+
+#define MASK(x) (BIT(x) - 1)
+#define ATU_PTEPTR_SHIFT 6
+#define PTE_ADDR(pte) ((pte).ptr << ATU_PTEPTR_SHIFT)
+#define pfn2phys __pfn_to_phys
+#define PHYS_TO_PAGE(pa) pfn_to_page(__phys_to_pfn(pa))
+
+#define C_ADDR_SHIFT 12
+#define ATU_PTE_ENTRY_SIZE    sizeof(union a_pte)
+#define flsl(x) (fls64(x) - 1)
+#define ffsl(x) (fls64((x) & -(x)) - 1)
+#define ATUCQ_TIMEOUT (odp_sw_decouple ? decouple_timeout_us : atucq_timeout_us)
+#define ATU_IBW_TIMEOUT (5 * MSEC_PER_SEC * USEC_PER_MSEC)
+#define ATU_ODPQ_SPACE_TIMEOUT (100 * USEC_PER_MSEC)
+#define ATU_STS_ATUCQ_IDLE_TIMEOUT (10 * MSEC_PER_SEC * USEC_PER_MSEC)
+#define ATU_STS_IBW_IDLE_TIMEOUT (1000 * MSEC_PER_SEC * USEC_PER_MSEC)
+#define ATUCQ_ENTRY(index) (index & (C_ATU_CFG_CQ_TABLE_ENTRIES - 1))
+/* Based on the AC base page size */
+#define ATU_PMD_ORDER(cac) (cac->huge_shift - cac->page_shift)
+#define ATU_PMD_NR(cac) BIT(ATU_PMD_ORDER(cac))
+#define ATU_PTE_ORDER(cac) (cac->page_shift - PAGE_SHIFT)
+#define ATU_PTE_NR(cac) BIT(ATU_PTE_ORDER(cac))
+#define PFN_INC(cac, leaf) (leaf ? ATU_PMD_NR(cac) : 1)
+/* Based on the kernel base page size */
+#define K_PMD_ORDER(cac) (cac->huge_shift - PAGE_SHIFT)
+#define K_PMD_NR(cac) BIT(K_PMD_ORDER(cac))
+#define KPFN_INC(cac, leaf) (leaf ? K_PMD_NR(cac) : ATU_PTE_NR(cac))
+#define ATU_FLAGS_MASK ~(CXI_MAP_READ | CXI_MAP_WRITE | CXI_MAP_ALLOC_MD | \
+			 CXI_MAP_FAULT | CXI_MAP_PREFETCH)
+#define ATUCQ_INVALIDATE_ALL 0x200000000000000UL
+#define MAX_PG_TABLE_SIZE 15
+#define MIN_PG_TABLE_SIZE 3
+#define MIN_DEVICE_PG_TABLE_SIZE (MIN_PG_TABLE_SIZE + 2)
+#define INVALID_DMABUF_FD (-1)
+
+/* Reserve an AC for physical mapping to be shared by all processes */
+#define ATU_PHYS_AC (C_ATU_CFG_AC_TABLE_ENTRIES - 1)
+#define ATU_NTA_DEF_PI_ACXT ATU_PHYS_AC
+#define ATU_FLUSH_REQ_DELAY 0x3ff
+
+#ifndef VMA_ITERATOR
+#define for_each_vma(vmi, vma) for (; vma; vma = vma->vm_next)
+#endif /* VMA_ITERATOR */
+
+#define CASS_NUM_LE_POOLS (C_LPE_CFG_PE_LE_POOLS_ENTRIES / C_PE_COUNT)
+#define CASS_MIN_POOL_TLES 8
+#define ACS_AVAIL (ATU_PHYS_AC - 1)
+#define CTS_AVAIL (C_NUM_CTS - 1) /* CT 0 invalid */
+#define EQS_AVAIL (C_NUM_EQS - 1) /* EQ 0 invalid */
+
+#define CASS_MAX_IRQ_NAME 24
+
+/* Current devices have up to 31 sensors */
+#define CASS_MAX_SENSORS 32
+
+#define CASS2_MAX_PACKETS_INFLIGHT 128U
+#define CASS1_MAX_PACKETS_INFLIGHT 64U
+
+#define DEFAULT_LE_POOL_ID 0
+#define DEFAULT_TLE_POOL_ID 0
+
+enum cass_link_speed {
+	CASS_SPEED_UNKNOWN = PCI_SPEED_UNKNOWN,
+	CASS_SPEED_2_5GT = PCIE_SPEED_2_5GT,
+	CASS_SPEED_5_0GT = PCIE_SPEED_5_0GT,
+	CASS_SPEED_8_0GT = PCIE_SPEED_8_0GT,
+	CASS_SPEED_16_0GT = PCIE_SPEED_16_0GT,
+	CASS_SPEED_32_0GT = PCIE_SPEED_32_0GT,
+
+	CASS_SPEED_20_0GT,
+	CASS_SPEED_25_0GT,
+};
+
+/* Cassini clock frequency */
+#define C1_CLK_FREQ_HZ 1000000000ULL /* 1 GHz */
+#define C2_CLK_FREQ_HZ 1100000000ULL /* 1.1 GHz */
+
+extern const struct file_operations uc_fops;
+
+union a_pte {
+	struct c_page_table_entry pte;
+	u64 qw;
+};
+
+struct l1_entry {
+	/* The array of PTEs */
+	struct page *l1_pages;
+	dma_addr_t l1_pages_dma_addr;
+
+	/* Bitmap of used PTEs */
+	unsigned long *bitmap;
+};
+
+struct cass_nta {
+	/* nta root page table pointer virtual address */
+	union a_pte *root_ptr;
+	struct page *root_page;
+	dma_addr_t root_page_dma_addr;
+	int root_order;
+	int l1_order;
+	int numa_node;
+
+	/* Each L0 PDE has its own bitmap of allocated L1 PTE. */
+	int l0_entries;
+	int l1_entries;
+	struct l1_entry l1[];
+};
+
+struct cass_ac {
+	struct cxi_ac ac;
+	struct list_head list;
+	struct list_head md_list;
+	struct mutex md_mutex;
+	struct cxi_lni_priv *lni_priv;
+
+	struct mutex ac_mutex;
+
+	/* copy of the atu ac struct */
+	union c_atu_cfg_ac_table cfg_ac;
+	u32 flags;
+
+	int page_shift;
+	int huge_shift;
+	int ptg_mode;
+	bool hugepage_test;
+
+	struct cass_nta *nta;
+
+	u64 iova_base;
+	u64 iova_end;
+	u64 iova_len;
+
+	struct iova_domain *iovad;
+	struct dentry *debug_dir;
+	struct dentry *lni_dir;
+	refcount_t refcount;
+	refcount_t reader_refs;
+	struct completion readers_done;
+
+	u32 pasid;
+	struct iommu_sva *sva;
+};
+
+/* Value written by the ATU CQ in wait_rsp_data when completing
+ * commands.
+ */
+#define WAIT_RSP_DATA 0xf00fbeefULL
+
+struct cass_atu_cq {
+	/* Protects access to the ATU CQ and wait_rsp_data */
+	struct mutex atu_cq_mutex;
+	struct mutex atu_ib_mutex;
+	/* Inbound-wait hardware failure latch, protected by atu_ib_mutex. */
+	bool hw_failure;
+
+	struct wait_rsp_data {
+		u64 cmp ____cacheline_aligned;
+		u64 ib ____cacheline_aligned;
+	} wait_rsp_data;
+
+	dma_addr_t rsp_dma_addr;
+	dma_addr_t cmp_wait_rsp_addr;
+	dma_addr_t ib_wait_rsp_addr;
+
+	char cmpl_wait_int_name[CASS_MAX_IRQ_NAME];
+};
+
+struct sts_idle {
+	struct cass_dev *hw;
+	bool ib_wait;
+};
+
+#define MAX_VFMSG_SIZE (22432 + sizeof(struct vf_pf_msg_hdr))
+
+/* A VF to PF message header */
+struct vf_pf_msg_hdr {
+	unsigned int len;	/* whole message length */
+	int rc;			/* return code */
+	uid_t uid;
+	gid_t gid;
+	int seq;
+};
+
+/* Traffic Class descriptions. */
+struct cass_tc_cfg {
+	struct list_head tc_entry;
+
+	unsigned int tc;
+	enum cxi_traffic_class_type tc_type;
+
+	/* Restricted request DSCP. */
+	unsigned int res_req_dscp;
+
+	/* Unrestricted request DSCP. */
+	unsigned int unres_req_dscp;
+
+	/* Restricted response DSCP. */
+	unsigned int res_rsp_dscp;
+
+	/* Unrestricted response DSCP. */
+	unsigned int unres_rsp_dscp;
+
+	/* Request PCP. */
+	unsigned int req_pcp;
+
+	/* Response PCP. */
+	unsigned int rsp_pcp;
+
+	/* CQ OCU Set Index */
+	unsigned int ocuset;
+
+	/* Traffic class value specific to CQ configuration. */
+	unsigned int cq_tc;
+
+	/* Buffer Class */
+	unsigned int req_bc;
+};
+
+struct cass_rsrc_info {
+	u16 res;
+	u16 shared_total;
+	u16 shared_in_use;
+};
+
+struct cass_irq {
+	struct atomic_notifier_head nh;
+	int idx;
+	struct cpumask mask;
+	char irq_name[CASS_MAX_IRQ_NAME];
+	atomic_t refcount;
+	int vec;
+	struct mutex lock;
+};
+
+/* Maximum size of an error flag CSR. For Cassini-1, they are all 64
+ * bits. For Cassini-2, C_HNI_PML_ERR_FLG is 256 bits long.
+ */
+#define MAX_ERR_FLAG_BITLEN 256
+union err_flags {
+	DECLARE_BITMAP(mask, MAX_ERR_FLAG_BITLEN);
+	union c_pi_err_flg pi;
+	union c_pi_ext_err_flg pi_ext;
+	union c_pi_ipd_ext_err_flg pi_ipd_ext;
+	union c_atu_err_flg atu;
+	union c_atu_ext_err_flg atu_ext;
+	union c1_hni_err_flg c1_hni;
+	union c2_hni_err_flg c2_hni;
+	union c2_hni_ext_err_flg c2_hni_ext;
+	union c1_hni_pml_err_flg c1_hni_pml;
+	union ss2_port_pml_err_flg ss2_port_pml;
+	union c_rmu_err_flg rmu;
+	union c_lpe_err_flg lpe;
+	union c_pct_ext_err_flg pct_ext;
+	union c_ee_err_flg ee;
+	union c_ixe_err_flg ixe_err_flg;
+};
+
+struct cxi_reg_err_flg {
+	struct list_head list;
+	unsigned int irq;
+	bool is_ext;
+	union err_flags err_flags;
+	/* Callback. irq is the MSI-X interrupt index (0 to 29), and
+	 * is_ext indicates whether the flag is set in the ERR_FLG or
+	 * EXT_ERR_FLG CSR. bitn is the bit index in abovementioned
+	 * CSR.
+	 */
+	void (*cb)(struct cass_dev *hw, unsigned int irq,
+		   bool is_ext, unsigned int bitn);
+};
+
+/* For error CSR processing */
+struct flg_err_info;
+struct csr_info {
+	const char *csr_name;
+	const char *csr_name_lo; /* same, in lower case */
+
+	/* Various CSR offsets for that error. */
+	unsigned int flg;
+	unsigned int mask;
+	unsigned int clr;
+	unsigned int first_flg;
+	unsigned int first_flg_ts;
+	unsigned int bitlen;
+	const struct flg_err_info *flg_info;
+};
+
+extern const struct file_operations decouple_stats_fops;
+extern const struct file_operations sw_decouple_fops;
+
+/*
+ * there currently are not that many entities that need to reserve
+ * sets of dma descriptors.
+ *
+ * Currently, cassini telemetry/counters (with test/experimental code)
+ * uses five dmac descriptor sets and rosetta link monitoring (lmon)
+ * may use two dma descriptor sets.
+ *
+ */
+#define DMAC_DESC_SET_COUNT			16
+
+struct dmac_desc_set {
+	u16 count;      /* total number of descs */
+	u16 index;      /* index of first desc */
+	u16 numused;    /* number of descs written */
+	const char *name;	/* descriptive name to identify use */
+};
+
+struct cass_vf {
+	struct task_struct *task;
+	struct socket *req_sock;
+	struct socket *notif_sock;
+
+	/* Index and back-pointer to hardware struct for use by VF handler thread */
+	struct cass_dev *hw;
+	int vf_idx;
+
+	/* ID of physical-address context for use when configuring CQ, EQ, etc. */
+	unsigned int phys_ac;
+
+	unsigned int token;
+	int notif_seq;
+
+	/* Per VF mutex to maintain notification message/response sequencing */
+	struct mutex notif_lock;
+
+	/* Per-VF message buffers */
+	char request[MAX_VFMSG_SIZE];
+	char reply[MAX_VFMSG_SIZE];
+
+	struct task_struct *kvm_task;
+};
+
+enum cxi_rmu_eth_filter_mode {
+	CXI_RMU_ETH_FILTER_NONE = 0,
+	CXI_RMU_ETH_FILTER_DIRECT,
+	CXI_RMU_ETH_FILTER_RSS,
+};
+
+/**
+ * struct cxi_rmu_eth_priv - Private Ethernet resource tracking (INTERNAL)
+ *
+ * Full internal structure containing all resource management state.
+ * One allocation per function (PF or VF).
+ */
+struct cxi_rmu_eth_priv {
+	struct cxi_rmu_eth rmu_eth;     /* Public opaque handle */
+	struct cxi_dev *dev;            /* Device this allocation belongs to */
+	bool is_vf;                     /* True if this is a VF allocation */
+	unsigned int vf_num;            /* VF number (if is_vf=true) */
+	bool admin;                     /* True if it has admin privileges */
+
+	/* Resource allocation */
+	unsigned int indir_base;        /* Absolute offset in HW indirection table */
+	unsigned int indir_size;        /* Number of indir entries allocated */
+	unsigned int set_list_base;     /* Absolute offset in HW set_list (4+ for dynamic) */
+	unsigned int set_list_quota;    /* Max set_list entries for this function */
+
+	/* Default RSS configuration (used by all MACs when use_rss=true) */
+	unsigned int rss_queues;
+	struct cxi_pte *ptes[64];   /* PTE pointers for RSS queues */
+	u32 hash_types;
+	unsigned int indir_entries;     /* Active indirection table size */
+
+	/* Filter tracking [0..quota-1]: enum cxi_rmu_eth_filter_mode values */
+	u8 *mac_filter_slots;
+};
+
+struct cass_ac *cass_cac_get(struct cass_dev *hw, int acid);
+void cass_cac_put(struct cass_ac *cac);
+
+/* Private hardware data for Cassini. This is not seen by clients. */
+struct cass_dev {
+	/* Embed a cxi device. */
+	struct cxi_dev cdev;
+	struct cxi_cfg_group_item *cfg_dev_dir;
+
+	/* Register base in BAR0 */
+	void __iomem *regs;
+	unsigned long regs_base;
+
+	/* /sys/class interface */
+	struct device class_dev;
+
+	/* Device type and revision */
+	union c_mb_sts_rev rev;
+	bool reduced_pct_tables;
+
+	/* TODO: For development only. Netsim VFs are not really
+	 * supported.
+	 */
+	bool with_vf_support;
+
+	/* Number of VFs currently configured in the PF. */
+	unsigned int num_vfs;
+
+	/* Interrupt vector used for the PF/ VF communication */
+	unsigned int pf_vf_vec;
+	char pf_vf_int_name[CASS_MAX_IRQ_NAME];
+	struct completion pf_to_vf_comp;
+
+	/* Virtual function communication socket for SR-IOV. Used by PF to
+	 * listen for incoming VF connections, used by VF to initiate connection
+	 */
+	struct socket *vf_req_sock;
+	struct socket *vf_notif_sock;
+	struct task_struct *vf_listener;
+	struct task_struct *vf_notif_handler;
+	u8 vf_notif_buf[MAX_VFMSG_SIZE];
+	u8 vf_notif_rsp_buf[MAX_VFMSG_SIZE];
+	struct cass_vf vfs[C_NUM_VFS];
+	cxi_msg_relay_t msg_relay;
+	void *msg_relay_data;
+	struct mutex msg_relay_lock;
+	unsigned int vf_cmd_seq;
+	/* Mutex to maintain command message/response sequencing */
+	struct mutex vf_cmd_lock;
+
+	/* PCIe info */
+	bool esm_active;
+	u8 pcie_link_speed;
+	bool pci_disabled;
+
+	struct {
+		/* Monitoring */
+		struct delayed_work task;
+
+		/* Acceptable number of correctable errors per minute */
+		unsigned int corr_err_min;
+
+		/* PCIe Vendor Specific Information position */
+		unsigned int event_counter_control;
+		unsigned int event_counter_data;
+
+		/* Number of correctable and uncorrectable errors seen. */
+		u64 uncorr_err;
+		u64 corr_err;
+	} pcie_mon;
+
+	/* Error interrupts handling. Represent each of the 30 error
+	 * interrupts by a bit in err_irq_mask.
+	 */
+	unsigned long err_irq_mask;
+	unsigned long err_irq_raised;
+	int err_irq_vecs[NUM_ERR_INTS];
+	char err_int_names[NUM_ERR_INTS][CASS_MAX_IRQ_NAME];
+	struct workqueue_struct *err_irq_wq;
+	struct work_struct err_irq_work;
+	struct kobject *err_flgs_dir_kobj[2]; /* IRQA and IRQB */
+	const struct csr_info (*err_handlers)[NUM_ERR_INTS][2];
+
+	spinlock_t sfs_err_flg_lock; /* protects mask update */
+	struct sfs_err_flg {
+		struct kobject kobj;
+
+		/* IRQ A mask for the error flag CSR. */
+		DECLARE_BITMAP(err_irq_mask, MAX_ERR_FLAG_BITLEN);
+		unsigned int bitlen;
+
+		/* Which bits must never be automatically masked */
+		union err_flags no_auto_mask;
+
+		/* Which bits must never be printed in logs. Some error bits
+		 * can be raised during normal operation (eg. UC_INTERRUPT)
+		 * and should not appear in logs.
+		 */
+		union err_flags no_print_mask;
+	} sfs_err_flg[NUM_ERR_INTS][2];
+
+	/* Clients registered with the error interrupt handler, for
+	 * certain bits
+	 */
+	struct mutex err_flg_mutex;
+	struct list_head err_flg_list;
+
+	/* Temporary storage for CSR error processing. As there is
+	 * only one work at any given time, there is no race to access
+	 * these.
+	 */
+	struct {
+		struct err_info {
+			u64 data[5];
+			int count;
+		} err_info[C_MAX_CSR_ERR_INFO];
+		unsigned int err_info_count;
+		u64 cntrs_val[C_MAX_CSR_CNTRS];
+		unsigned int err_cntrs_count;
+		char err_info_str[100];
+		DECLARE_BITMAP(first_flg, MAX_ERR_FLAG_BITLEN);
+		union c_any_err_first_flg_ts first_flg_ts;
+	};
+
+	/* Each error is rate limited. If too many interrupts happen
+	 * for the same error bit, then the bit will be permanently
+	 * masked.
+	 */
+	struct ratelimit_state err_rl[NUM_ERR_INTS][MAX_ERR_FLAG_BITLEN];
+
+	/* Debugfs directories for that device */
+	struct dentry *debug_dir;
+	struct dentry *port_dir;
+	struct dentry *stats_dir;
+	struct dentry *lni_dir;
+	struct dentry *atu_dir;
+	struct dentry *decouple_dir;
+
+	/* Count users of this device. */
+	refcount_t refcount;
+
+	/* Keep track of NIs. TODO: keep as that may help debugging,
+	 * but should be removed eventually.
+	 */
+	struct list_head lni_list;
+	spinlock_t lni_lock;
+
+	struct {
+		atomic_t lni;
+		atomic_t domain;
+		atomic_t eq;
+		atomic_t txq;
+		atomic_t tgq;
+		atomic_t pt;
+		atomic_t ct;
+		atomic_t ac;
+		atomic_t md;
+	} stats;
+
+	struct dentry *domain_dir;
+	struct dentry *eq_dir;
+	struct dentry *cq_dir;
+	struct dentry *pt_dir;
+	struct dentry *ct_dir;
+	struct dentry *ac_dir;
+
+	/* Pending LNI cleanups */
+	struct list_head lni_cleanups_list;
+	struct delayed_work lni_cleanups_work;
+	struct mutex lni_cleanups_lock;
+
+	/* Number of IRQs available to that device. */
+	unsigned int num_irqs;
+
+	unsigned int atu_cq_vec;
+	unsigned int atu_pri_vec;
+
+	/* Completion IRQs */
+	unsigned int num_comp_irqs;
+	struct cass_irq *comp_irqs;
+
+	/* RMU table allocation bitmap. Each entry in the RMU table
+	 * contains a different VNI.
+	 */
+	struct ida rmu_index_table;
+	struct ida domain_table;
+	spinlock_t rmu_lock;
+	struct rb_root rmu_tree;
+
+	/* Physical PID leases and per-profile use counts are protected by
+	 * pid_pool_lock. domain_lock protects unique VNI+PID domain identities.
+	 */
+	spinlock_t pid_pool_lock;
+	spinlock_t domain_lock;
+	struct rb_root domain_tree;
+	DECLARE_BITMAP(leased_pids, 1 << MAX_PID_BITS);
+	u32 pid_usecount[1 << MAX_PID_BITS];
+
+	/* EQ allocation bitmap */
+	spinlock_t eq_shadow_lock;
+	struct ida eq_index_table;
+	struct mutex init_eq_hw_state;
+
+	/* Which EQ is registered with the PCT block. Protected by
+	 * init_eq_hw_state.
+	 */
+	unsigned int pct_eq_n;
+
+	/* Communication Profile Table (CPT) and containers. The table,
+	 * cpt_index_table, resides on the adapter and is used to reserve
+	 * one of the hardware resource. The RB tree, cp_cont_tree, is used to
+	 * store the allocated communication containers. Each container is
+	 * identified by a VNI and traffic class. Each container may contain
+	 * multiple communication profiles which map to an index in the CPT.
+	 * The lock, cp_lock, protects the tree.
+	 */
+	struct ida cp_table;
+	struct mutex cp_lock;
+
+	/* Configured traffic classes; entries can be added and removed at run time. */
+	struct list_head tc_list;
+	struct mutex tc_lock;	/* protects tc_list */
+
+	/* Next resource to be used for traffic class configuration. */
+	unsigned int next_oxe_bc;
+	unsigned int next_oxe_branch_bucket;
+	unsigned int next_oxe_leaf_bucket;
+	unsigned int next_cq_tc;
+	unsigned int next_cq_ocuset;
+	unsigned int next_cq_ocu;
+	unsigned int next_ixe_wrq;
+	unsigned int next_ixe_fq;
+	unsigned int next_lpe_fq;
+
+	u8 tsc_fill_qty;
+	struct qos_profile qos;
+	struct kobject tcs_kobj;
+
+	/* CQ allocation bitmaps. */
+	spinlock_t cq_shadow_lock;
+	struct mutex cq_init_lock;
+	struct ida cq_table;
+
+	/* CT allocations */
+	struct mutex ct_init_lock;
+
+	/* Multicast uniqueness table */
+	struct ida multicast_table;
+
+	/* PT allocation bitmap */
+	spinlock_t lpe_shadow_lock;
+	struct ida pte_table;
+	struct ida pt_index_table;
+	unsigned int dmac_pt_id;
+	struct mutex mst_table_lock;
+	struct mutex pte_transition_sm_lock;
+	union c_mst_dbg_mst_table *mst_entries;
+	dma_addr_t mst_entries_dma_addr;
+	atomic_t plec_count;
+
+	/* LE pool allocation bitmap */
+	struct ida le_pool_ids[C_PE_COUNT];
+	struct ida tle_pool_ids;
+
+	/* Ethernet Set List */
+	struct ida set_list_table;
+	struct cxi_tx_profile eth_tx_profile;
+
+	/* RMU Ethernet resource management */
+	struct idr rmu_eth_idr;
+
+	/* Protects all RMU Eth resources */
+	struct mutex rmu_eth_lock;
+
+	/* Protects C_RMU_CFG_PORTAL_LIST_X/Y. */
+	spinlock_t rmu_portal_list_lock;
+
+	/* RGID management */
+	struct xarray rgid_array;
+	refcount_t rgids_refcount;
+	struct ida lni_table;
+
+	/* ATU info */
+	void *oxe_dummy_addr;
+	dma_addr_t oxe_dummy_dma_addr;
+	struct cass_ac *cac_table[C_ATU_CFG_AC_TABLE_ENTRIES];
+	/* Protects AC publication and acquisition of reader pins. */
+	spinlock_t cac_table_lock;
+	struct ida atu_table;
+	struct cass_atu_cq atu_cq;
+	spinlock_t atu_shadow_lock;
+	atomic_t atu_error_inject;
+	atomic_t atu_odp_requests;
+	atomic_t atu_odp_fails;
+	atomic_t atu_prb_expired;
+	char odp_pri_int_name[CASS_MAX_IRQ_NAME];
+	int pri_rd_ptr;
+	struct work_struct pri_work;
+	struct workqueue_struct *pri_wq;
+	struct c_page_request_entry *page_request_table;
+	dma_addr_t prt_dma_addr;
+	struct cxi_reg_err_flg atu_err;
+	struct work_struct prb_work;
+	struct workqueue_struct *prb_wq;
+	struct mutex odpq_mutex;
+	struct list_head mm_lni_list;
+	struct mutex mm_mutex;
+	struct ida md_index_table;
+	bool ac_filter_disabled;
+	/* Flag to indicate that the IOMMU callback is in place to support
+	 * C1 ODP.
+	 */
+	bool ats_c1_odp_enable;
+
+	/* Number of invalidations due to memory deregistration */
+	atomic_t dcpl_md_clear_inval;
+	/* Number of invalidations due to the mmu notifier for NTA mode */
+	atomic_t dcpl_nta_mn_inval;
+	/* Number of invalidations due to the mmu notifier (via IOMMU callback)
+	 * for ATS mode
+	 */
+	atomic_t dcpl_ats_mn_inval;
+	/* Completion wait count */
+	atomic_t dcpl_comp_wait;
+	/* ODP decoupling entered - Completion wait has timed out */
+	atomic_t dcpl_entered;
+	/* ixe_epoch_x_cntr was 0 */
+	atomic_t dcpl_ixe_cntr_0;
+	/* ixe_epoch_x_cntr was not initially 0 but it decremented to 0 */
+	atomic_t dcpl_ixe_cntr_dec_0;
+	/* ee_epoch_x_cntr count of entering dec loop */
+	atomic_t dcpl_ee_cntr_dec_0;
+	/* ee_epoch_x_cntr is stuck */
+	atomic_t dcpl_ee_cntr_stuck;
+	/* oxe_epoch_x_cntr count of entering dec loop */
+	atomic_t dcpl_oxe_cntr_dec_0;
+	/* oxe_epoch_x_cntr is stuck */
+	atomic_t dcpl_oxe_cntr_stuck;
+	/* ixe_epoch_x_cntr is stuck after hw decoupling enabled */
+	atomic_t dcpl_ixe_cntr_stuck;
+	/* decouple process successful after hw decoupling enabled */
+	atomic_t dcpl_success;
+	/* ib_epoch_x_cntr decremented to 0 in step 6.2 */
+	atomic_t dcpl_ibw_cntr_dec_0_62;
+	/* ib_epoch_x_cntr count of dec loops in step 6.2 */
+	atomic_t dcpl_ibw_cntr_dec_0_62_count;
+	/* ib_epoch_x_cntr decremented to 0 in step 6.4 */
+	atomic_t dcpl_ibw_cntr_dec_0_64;
+	/* ib_epoch_x_cntr count of dec loops in step 6.4 */
+	atomic_t dcpl_ibw_cntr_dec_0_64_count;
+	/* ib_epoch_x_cntr is stuck */
+	atomic_t dcpl_step7;
+	/* IBW is active and ib_epoch_x_cntr is stuck */
+	atomic_t dcpl_ibw_active_stuck;
+	/* How long waiting to get the lock */
+	atomic_t dcpl_ibw_idle_wait;
+	/* IBW counter is inside of trylock loop */
+	atomic_t dcpl_ibw_cntr_is_0;
+	/* Decouple issued inbound wait */
+	atomic_t dcpl_ibw_issued;
+	/* Maximum time spent in decoupling */
+	ktime_t dcpl_max_time;
+	/* Histogram of time spent in decoupling */
+	#define DBINS 12
+	#define FIRST_BIN (64 * 1024)
+	int dcpl_time[DBINS];
+	/* Maximum time spent finding an MD */
+	ktime_t pri_max_md_time;
+	/* Histogram of time spent in finding MD during a fault */
+	#define MDBINS 8
+	#define FIRST_MDBIN (8 * 1024)
+	int pri_md_time[MDBINS];
+	/* Maximum time spent servicing a fault */
+	ktime_t pri_max_fault_time;
+	/* Histogram of time spent servicing a fault */
+	#define FBINS 7
+	#define FIRST_FBIN (8 * 1024)
+	int pri_fault_time[FBINS];
+
+	/* Counting event bitmap. */
+	struct ida ct_table;
+
+	/* Precision Time Protocol Clock */
+	struct ptp_clock_info ptp_info;
+	struct ptp_clock *ptp_clock;
+	spinlock_t rtc_lock;
+
+	/* MST locks. */
+	spinlock_t mst_match_done_lock;
+	struct mutex mst_update_lock;
+
+	/* Micro-controller */
+	bool uc_present;
+	int uc_platform;	/* CUC_BOARD_TYPE_xxx */
+	struct cxi_reg_err_flg uc_err;
+	struct completion uc_attention0_comp;
+	struct cuc_pkt uc_req;
+	struct cuc_pkt uc_resp;
+	struct workqueue_struct *uc_attn1_wq;
+	struct work_struct uc_attn1_work;
+	struct mutex uc_mbox_mutex;
+	struct i2c_adapter uc_i2c;
+	bool uc_i2c_locked;
+	s8 uc_nic;		/* NIC index from uC point of view */
+	u8 default_mac_addr[ETH_ALEN];
+	struct kobject uc_kobj;
+	char *fw_versions[7];	/* indexed by enum casuc_fw_target */
+	bool fru_info_valid;	/* FRU query was successful; do not query again. */
+	char *fru_info[PLDM_FRU_FIELD_VENDOR_IANA + 1];
+	struct idr pldm_sensors;
+	unsigned int pldm_sensors_count;
+	struct kobject *pldm_sensors_kobj;
+	unsigned int uc_dmac_id;
+	struct kobject port_kobj;
+	struct kobject port_num_kobj;
+	bool qsfp_beacon_active;
+	bool qsfp_over_temp;
+	bool qsfp_bad;
+
+	/* HW mon */
+	struct {
+		struct device *dev;
+		struct hwmon_chip_info chip_info;
+		const struct hwmon_channel_info *all_types[hwmon_max];
+
+		struct hwmon_channel_info info[5];
+
+		/* Enough room for 30 sensors. */
+		u32 config[CASS_MAX_SENSORS + hwmon_max];
+	} hwmon;
+
+	/* RX and TX Profiles */
+	struct cxi_rxtx_profile_list rx_profile_list;
+	struct cxi_rxtx_profile_list tx_profile_list;
+	struct mutex tx_profile_get_lock;
+	struct mutex rx_profile_get_lock;
+
+	/* Services */
+	struct mutex svc_lock;
+	struct idr svc_ids;
+	struct dentry *svc_debug;
+	struct list_head svc_list;
+	unsigned int svc_count;
+	struct cxi_resource_use resource_use[CXI_RESOURCE_MAX];
+	spinlock_t rgrp_lock;
+
+	/* Resource Groups */
+	struct cxi_rgroup_list rgroup_list;
+
+	/* HNI */
+	struct cxi_reg_err_flg hni_pause_err;
+	struct work_struct pause_timeout_work;
+	struct cxi_reg_err_flg hni_uncor_err;
+	struct cxi_reg_err_flg hni_pml_uncor_err;
+
+	/* IXE */
+	struct cxi_reg_err_flg ixe_pbuf_rd_err;
+
+	/* PML */
+	struct cxi_reg_err_flg pml_err;
+	struct {
+		spinlock_t lock;
+		enum cass_phy_state state;
+		struct delayed_work state_queue;
+		/* protected by lock; bumped by every non-worker transition */
+		unsigned int generation;
+		bool stopped;
+	} phy;
+
+	u32 max_eth_rxsize;
+
+	/* Ethernet TX timestamp shift. 0 to 3, stays at zero but
+	 * could become a kernel parameter.
+	 */
+	int tx_timestamp_shift;
+
+	/* base link support */
+	struct sbl_inst *sbl;           /* slingshot base-link instance */
+	atomic_t *sbl_counters;         /* sbl link counters */
+	bool link_config_dirty;         /* link config != running config */
+	spinlock_t sbl_state_lock;      /* link state lock */
+	struct cass_port *port;         /* port db array */
+	const struct cxi_link_ops *link_ops;
+
+	/* SBus config */
+	struct mutex sbus_mutex;      /* serialise SBus config accesses */
+
+	/* Rendezvous get control config */
+	struct mutex get_ctrl_mutex;  /* serialize writes from sysfs */
+
+	/* AMO remap sysfs mutex. */
+	struct mutex amo_remap_to_pcie_fadd_mutex;
+
+	/* sysfs support */
+	struct kobject properties_kobj;
+	struct kobject fru_kobj;
+	struct kobject link_restarts_kobj;
+
+	/* debugfs support */
+	struct attribute_group  port_group;
+	struct attribute        **port_attrs;
+	struct device_attribute *port_dev_attrs;
+	struct device_attribute port_attr;
+
+	/* Current copy of the QSFP eeprom, with its length. If length
+	 * is 0, the data is not valid.
+	 */
+	struct mutex qsfp_eeprom_lock;
+	u8 qsfp_eeprom_page0[ETH_MODULE_SFF_8436_LEN];
+	u8 qsfp_eeprom_page1[ETH_MODULE_SFF_8436_LEN];
+	u8 qsfp_format;
+	unsigned int qsfp_eeprom_page_len;
+
+	/*
+	 * telemetry for cassini
+	 */
+	struct {
+		struct telem_info	*info;
+	} telemetry;
+
+	/*
+	 * dma controller
+	 */
+	struct {
+		struct mutex			lock;
+		struct completion		irupt;
+		char				irupt_name[CASS_MAX_IRQ_NAME];
+		struct c_pi_dmac_cdesc		*cpl_desc;
+		dma_addr_t                      cpl_desc_dma_addr;
+		struct dmac_desc_set		*desc_sets;
+		unsigned long			*desc_map;
+	} dmac;
+
+	/* sl */
+	struct cass_sl_dev sl;
+
+	/*auxiliary device*/
+	void *cxi_adev;
+	/*
+	 * Protects the publication of cxi_adev and covers every use of it, so
+	 * the auxiliary device cannot be freed while a NID change is being
+	 * sent to it.
+	 */
+	struct mutex cxi_adev_lock;
+};
+
+static inline struct cass_dev *cxi_to_cass_dev(struct cxi_dev *dev)
+{
+	return container_of(dev, struct cass_dev, cdev);
+}
+
+/* Communication profile. */
+struct cass_cp {
+	struct cass_dev *hw;
+	struct cxi_tx_profile *tx_profile;
+
+	/* VNI/PCP and TC identify a unique communication profile. */
+	unsigned int vni_pcp;
+	enum cxi_traffic_class tc;
+	enum cxi_traffic_class_type tc_type;
+
+	/* Communication profile table ID. */
+	unsigned int id;
+
+	/* CP list ID. */
+	unsigned int list_id;
+
+	/* Number of users of this profile. */
+	refcount_t ref;
+};
+
+struct pldm_sensor {
+	struct cass_dev *hw;
+	unsigned int id;
+	struct kobject kobj;
+	char name[AUX_NAME_MAX];
+	struct numeric_sensor_pdr num;
+
+	/* Mapping when inserted into hwmon */
+	struct {
+		enum hwmon_sensor_types type;
+		int channel;
+		int multiplier;
+	} hwmon;
+};
+
+/* From struct getsensorreading_rsp */
+struct pldm_sensor_reading {
+	u8 operational_state;
+	u8 present_state;
+	u8 previous_state;
+	s64 present_reading;
+};
+
+/* Read one of the fields from a PLDM declaration */
+#define get_pldm_value(sensor, field) ({				\
+	const int s = sensor->num.sensor_data_size;			\
+	s64 value;							\
+									\
+	if (s == PLDM_DATA_SIZE_UINT8 || s == PLDM_DATA_SIZE_SINT8)	\
+		value = sensor->num.ssd.ssd8.field;			\
+	else if (s == PLDM_DATA_SIZE_UINT16 ||				\
+		 s == PLDM_DATA_SIZE_SINT16)				\
+		value = sensor->num.ssd.ssd16.field;			\
+	else								\
+		value = sensor->num.ssd.ssd32.field;			\
+									\
+	value;								\
+})
+
+static inline void __iomem *cass_csr(struct cass_dev *hw, u64 offset)
+{
+	BUG_ON(offset > C_MEMORG_CSR_SIZE);
+	return hw->regs + offset;
+}
+
+/**
+ * cass_clear() - Writes zeroes to a CSR
+ *
+ * @hw: the device
+ * @offset: offset of the CSR in the region
+ * @len: length to clear, in bytes. Must be a multiple of 8.
+ */
+static inline void cass_clear(struct cass_dev *hw, u64 offset, size_t len)
+{
+	void __iomem *base = cass_csr(hw, offset);
+	int i;
+
+	len /= sizeof(u64);
+
+	for (i = 0; i < len; i++, base += sizeof(u64))
+		writeq(0, base);
+}
+
+/* must come after cass_read/write/clear definitions. */
+#include <linux/hpe/cxi/libcassini.h>
+
+static inline bool cass_version(const struct cass_dev *hw,
+				enum cassini_version version)
+{
+	return cassini_version(&hw->cdev.prop, version);
+}
+
+static inline void cass_cond_lock(struct mutex *lock, bool need_lock)
+{
+	if (need_lock)
+		mutex_lock(lock);
+}
+
+static inline void cass_cond_unlock(struct mutex *lock, bool need_lock)
+{
+	if (need_lock)
+		mutex_unlock(lock);
+}
+
+static inline void cass_cp_lock(struct cass_dev *hw)
+{
+	mutex_lock(&hw->cp_lock);
+}
+
+static inline void cass_cp_unlock(struct cass_dev *hw)
+{
+	mutex_unlock(&hw->cp_lock);
+}
+
+static const char * const cxi_tc_strs[] = {
+	[CXI_TC_DEDICATED_ACCESS]       = "DEDICATED_ACCESS",
+	[CXI_TC_LOW_LATENCY]            = "LOW_LATENCY",
+	[CXI_TC_BULK_DATA]              = "BULK_DATA",
+	[CXI_TC_BEST_EFFORT]            = "BEST_EFFORT",
+	[CXI_TC_ETH]                    = "ETH",
+};
+
+static const char * const cxi_tc_type_strs[] = {
+	[CXI_TC_TYPE_DEFAULT] = "DEFAULT",
+	[CXI_TC_TYPE_HRP] = "HRP",
+	[CXI_TC_TYPE_RESTRICTED] = "RESTRICTED",
+	[CXI_TC_TYPE_COLL_LEAF] = "COLL_LEAF",
+};
+
+static inline const char *cxi_tc_to_str(enum cxi_traffic_class tc)
+{
+	if (tc >= CXI_TC_DEDICATED_ACCESS && tc < CXI_TC_MAX)
+		return cxi_tc_strs[tc];
+	return "(invalid)";
+}
+
+static inline
+const char *cxi_tc_type_to_str(enum cxi_traffic_class_type tc_type)
+{
+	if (tc_type >= CXI_TC_TYPE_DEFAULT && tc_type < CXI_TC_TYPE_MAX)
+		return cxi_tc_type_strs[tc_type];
+	return "(invalid)";
+}
+
+static const char * const cxi_rsrc_type_strs[] = {
+	[CXI_RSRC_TYPE_PTE] = "PTEs",
+	[CXI_RSRC_TYPE_TXQ] = "TXQs",
+	[CXI_RSRC_TYPE_TGQ] = "TGQs",
+	[CXI_RSRC_TYPE_EQ] = "EQs",
+	[CXI_RSRC_TYPE_CT] = "CTs",
+	[CXI_RSRC_TYPE_LE] = "LEs",
+	[CXI_RSRC_TYPE_TLE] = "TLEs",
+	[CXI_RSRC_TYPE_AC] = "ACs",
+};
+
+static inline
+const char *cxi_rsrc_type_to_str(enum cxi_rsrc_type rsrc_type)
+{
+	if (rsrc_type >= CXI_RSRC_TYPE_PTE && rsrc_type < CXI_RSRC_TYPE_MAX)
+		return cxi_rsrc_type_strs[rsrc_type];
+	return "(invalid)";
+}
+
+int cass_ixe_set_amo_remap_to_pcie_fadd(struct cass_dev *hw, int amo_remap);
+void cass_pte_init(struct cass_dev *hw);
+void cass_pte_set_get_ctrl(struct cass_dev *hw);
+int cass_atu_init(struct cass_dev *hw);
+void cass_atu_fini(struct cass_dev *hw);
+void cass_acs_disable(struct cxi_lni_priv *lni_priv);
+void cass_acs_free(struct cxi_lni_priv *lni_priv);
+int cass_ac_phys_alloc(struct cass_dev *hw, bool vf_en, int vf_num);
+void cass_ac_phys_free(struct cass_dev *hw, int acid);
+
+void cass_cq_init(struct cass_dev *hw);
+void cass_ee_init(struct cass_dev *hw);
+int cass_svc_init(struct cass_dev *hw);
+void cass_svc_fini(struct cass_dev *hw);
+int cass_tc_find(struct cass_dev *hw, enum cxi_traffic_class tc,
+		 enum cxi_traffic_class_type tc_type, u8 ethernet_pcp,
+		 struct cass_tc_cfg *tc_cfg);
+int cass_tc_init(struct cass_dev *hw);
+void cass_tc_fini(struct cass_dev *hw);
+void traffic_shaping_cfg(struct cass_dev *hw);
+void cass_tc_set_tx_pause_all(struct cass_dev *hw, bool enable);
+void cass_tc_set_rx_pause_all(struct cass_dev *hw, bool enable);
+void cass_clear_cps(struct cxi_tx_profile *tx_profile);
+
+static inline bool is_vni_valid(unsigned int vni)
+{
+	/* A VNI is 16 bits and VNI 0 is invalid */
+	return vni && !(vni & ~0xffff);
+}
+
+int register_error_handlers(struct cass_dev *hw);
+void deregister_error_handlers(struct cass_dev *hw);
+int cass_sriov_configure(struct pci_dev *pdev, int num_vfs);
+int cass_vf_init(struct cass_dev *hw);
+void cass_vf_fini(struct cass_dev *hw);
+
+#define ATU_CFG_AC_TABLE_MB_SHIFT 15
+/* Based on note in MEM_SIZE description - MEM_BASE must be aligned to
+ * 8x the largest configured page size.
+ * Derivative 1 Page Size as described in Table 364 - Page Table Sizes
+ */
+#define ATU_IOVA_BASE_SHIFT(ac) (12 + ac.base_pg_size + ac.pg_table_size + 3)
+#define ATU_IOVA_MASK(ac) (~MASK(ATU_IOVA_BASE_SHIFT(ac)))
+/* Maximum addressable memory range (56 bits) of Cassini. */
+#define ATU_MAX_ADDR_MEM_BITS 56
+#define ATU_LAC_SHIFT (ATU_MAX_ADDR_MEM_BITS - 3)
+#define ATU_VA_MASK(va) (va & MASK(ATU_MAX_ADDR_MEM_BITS))
+#define IS_DEVICE_MEM(md_priv) (md_priv->flags & CXI_MAP_DEVICE)
+
+extern struct mutex dev_list_mutex;
+extern int more_debug;
+extern struct genl_family cxierr_genl_family;
+extern struct cass_p2p_ops p2p_ops;
+extern bool odp_sw_decouple;
+extern bool ats_c1_override;
+
+/* Add the LAC to the top 3 bits of the IOVA so they are unique */
+static inline u64 cass_iova_base(struct cass_ac *cac)
+{
+	u64 iova_base;
+
+	iova_base = (u64)get_random_u32() << ATU_IOVA_BASE_SHIFT(cac->cfg_ac);
+	iova_base &= ~(0x7UL << ATU_LAC_SHIFT);
+	iova_base |= (u64)cac->ac.lac << ATU_LAC_SHIFT;
+
+	return iova_base;
+}
+
+void __iomem *cass_csr(struct cass_dev *hw, u64 offset);
+int cass_nta_cq_init(struct cass_dev *hw);
+void cass_nta_cq_fini(struct cass_dev *hw);
+int cass_nta_pri_init(struct cass_dev *hw);
+int cass_sts_idle(struct sts_idle *sidle);
+void cass_nta_pri_fini(struct cass_dev *hw);
+int cass_inbound_wait(struct cass_dev *hw, bool wait_for_response);
+u64 cass_nta_iova_alloc(struct cass_ac *cac, const struct ac_map_opts *m_opts);
+void cass_nta_iova_fini(struct cass_ac *cac);
+int cass_nta_iova_init(struct cass_ac *cac, size_t va_len);
+void cass_nta_free(struct cass_dev *hw, struct cass_nta *nta);
+void cass_l1_tables_free(struct cass_dev *hw, struct cass_nta *nta);
+void cass_clear_range(const struct cxi_md_priv *md_priv, u64 iova, u64 len);
+void cass_md_clear(struct cxi_md_priv *md_prv, bool inval, bool need_lock);
+void cass_dma_unmap_pages(struct cxi_md_priv *md_priv);
+int cass_dma_addr_mirror(dma_addr_t dma_addr, u64 iova, struct cass_ac *cac,
+			 u32 flags, bool is_huge_page, bool *invalidate);
+int cass_mirror_fault(const struct ac_map_opts *m_opts, u64 *pfns, int count,
+		      uintptr_t addr, size_t len);
+int cass_mirror_odp(const struct ac_map_opts *m_opts, struct cass_ac *cac,
+		    int npfns, u64 addr);
+int cass_mirror_device(struct cxi_md_priv *md_priv,
+		       const struct ac_map_opts *m_opts);
+int cass_mirror_range(struct cxi_md_priv *md_priv, struct ac_map_opts *m_opts);
+void cass_invalidate_range(const struct cass_ac *cac, u64 iova,
+			   size_t length);
+int cass_nta_init(struct cxi_lni_priv *lni_priv, struct ac_map_opts *m_opts,
+		  struct cass_ac *cac);
+int cass_nta_mirror_sgt(struct cxi_md_priv *md_priv, bool need_lock);
+int cass_nta_mirror_kern(struct cxi_md_priv *md_priv,
+			 const struct iov_iter *iter, bool need_lock);
+int cass_pin(const struct cass_ac *cac, struct page **pages, int npages,
+	     u64 addr, bool write);
+int cass_mirror_hp(const struct ac_map_opts *m_opts, struct cxi_md_priv *md_priv);
+int cass_pfns_mirror(struct cxi_md_priv *md_priv, const struct ac_map_opts *m_opts,
+		     u64 *pfns, int npfns, bool is_huge_page);
+int cass_vma_write_flag(struct mm_struct *mm, ulong start, ulong end,
+			u32 flags);
+int cass_pin_mirror(struct cxi_md_priv *md_priv, struct ac_map_opts *m_opts);
+void cass_notifier_cleanup(struct cxi_md_priv *md_priv);
+int cass_odp_decouple(struct cass_dev *hw);
+int cass_odp_supported(struct cass_dev *hw, u32 flags);
+int cass_mmu_notifier_insert(struct cxi_md_priv *md_priv,
+			     const struct ac_map_opts *m_opts);
+void cass_device_hugepage_size(int contig_cnt, struct ac_map_opts *m_opts);
+void cass_align_start_len(struct ac_map_opts *m_opts, uintptr_t va, size_t len,
+			  int align_shift);
+int cass_cpu_page_size(struct cass_dev *hw, struct ac_map_opts *m_opts,
+		       struct mm_struct *mm, uintptr_t va,
+		       const struct cxi_md_hints *hints, int *align_shift);
+int cass_ptp_init(struct cass_dev *hw);
+void cass_ptp_fini(struct cass_dev *hw);
+int cass_ats_init(struct cxi_lni_priv *lni_priv, struct ac_map_opts *m_opts,
+		  struct cass_ac *cac);
+int cass_ats_md_init(struct cxi_md_priv *md_priv, const struct ac_map_opts *m_opts);
+void cass_amd_iommu_inval_cb_init(struct pci_dev *pdev);
+void cass_unbind_ac(struct cass_dev *hw, const struct cass_ac *cac);
+void cass_iommu_init(struct cass_dev *hw);
+void cass_iommu_fini(struct cass_dev *hw);
+
+void cass_hni_init(struct cass_dev *hw);
+void cass_hni_fini(struct cass_dev *hw);
+int cass_cable_scan(struct cass_dev *hw);
+void update_oxe_link_up(struct cass_dev *hw);
+void update_hni_link_up(struct cass_dev *hw);
+int cass_create_tc_sysfs(struct cass_dev *hw);
+void cass_destroy_tc_sysfs(struct cass_dev *hw);
+
+int cass_ixe_init(struct cass_dev *hw);
+void cass_ixe_fini(struct cass_dev *hw);
+
+int cass_register_uc(struct cass_dev *hw);
+void cass_unregister_uc(struct cass_dev *hw);
+int cass_register_port(struct cass_dev *hw);
+void cass_unregister_port(struct cass_dev *hw);
+int uc_cmd_get_nic_id(struct cass_dev *hw);
+int uc_cmd_get_mac(struct cass_dev *hw);
+int uc_cmd_get_fru(struct cass_dev *hw);
+void uc_cmd_update_ier(struct cass_dev *hw, u32 set_bits, u32 clear_bits);
+int uc_cmd_qsfp_write(struct cass_dev *hw, u8 page, u8 addr,
+		      const u8 *data, size_t data_len);
+void uc_prepare_comm(struct cass_dev *hw);
+int uc_wait_for_response(struct cass_dev *hw);
+int cxi_get_qsfp_data(struct cxi_dev *cdev, u32 offset,
+		      u32 len, u32 page, u8 *data);
+void uc_cmd_set_link_leds(struct cass_dev *hw, enum casuc_led_states led_state);
+int update_sensor(struct pldm_sensor *sensor,
+		  struct pldm_sensor_reading *result);
+
+void finalize_pt_cleanups(struct cxi_lni_priv *lni, bool force);
+void finalize_cq_cleanups(struct cxi_lni_priv *lni, bool force);
+void finalize_eq_cleanups(struct cxi_lni_priv *lni);
+void finalize_ct_cleanups(struct cxi_lni_priv *lni);
+bool lni_cleanups(struct cass_dev *hw, bool force);
+void lni_cleanups_work(struct work_struct *work);
+void cass_rgid_init(struct cass_dev *hw);
+void cass_rgid_fini(struct cass_dev *hw);
+int cass_lac_get(struct cass_dev *hw, int id);
+void cass_lac_put(struct cass_dev *hw, int id, int lac);
+int cass_rgid_get(struct cass_dev *hw, struct cxi_rgroup *rgroup);
+void cass_rgid_put(struct cass_dev *hw, int id);
+int cass_lcid_get(struct cass_dev *hw, struct cxi_cp_priv *cp_priv, int rgid,
+		  enum cxi_trig_cp cp_type);
+void cass_lcid_put(struct cass_dev *hw, int rgid, int lcid);
+struct cxi_cp_priv *cass_cp_find(struct cass_dev *hw, int rgid, int lcid);
+struct cxi_cp_priv *cass_cp_rgid_find(struct cass_dev *hw,
+				      struct cxi_lni_priv *lni_priv,
+				      unsigned int vni_pcp, unsigned int tc,
+				      enum cxi_traffic_class_type tc_type,
+				      enum cxi_trig_cp cp_type);
+
+int get_dma_addr_if_contig(struct cass_dev *hw, void *addr, size_t len,
+			   enum dma_data_direction dir, struct page ***pages,
+			   dma_addr_t *dma_addr);
+
+int cass_telem_init(struct cass_dev *hw);
+void cass_telem_fini(struct cass_dev *hw);
+
+int create_sysfs_properties(struct cass_dev *hw);
+void destroy_sysfs_properties(struct cass_dev *hw);
+
+void cass_phy_init(struct cass_dev *hw);
+void cass_phy_start(struct cass_dev *hw, bool force_reconfig);
+void cass_phy_stop(struct cass_dev *hw, bool block);
+void cass_phy_bounce(struct cass_dev *hw);
+void cass_phy_trigger_machine(struct cass_dev *hw);
+void cass_phy_link_up(struct cass_dev *hw);
+void cass_phy_link_down(struct cass_dev *hw);
+
+extern unsigned int pe_total_les;
+void cass_cfg_le_pools(struct cass_dev *hw, int pool_id, int pe,
+		       const struct cxi_limits *les, bool release);
+void cass_cfg_tle_pool(struct cass_dev *hw, int pool_id,
+		       const struct cxi_limits *tles, bool release);
+void cass_tle_init(struct cass_dev *hw);
+
+int cass_dmac_init(struct cass_dev *hw);
+void cass_dmac_fini(struct cass_dev *hw);
+
+int cass_irq_init(struct cass_dev *hw);
+void cass_irq_fini(struct cass_dev *hw);
+struct cass_irq *cass_comp_irq_attach(struct cass_dev *hw,
+				      const struct cpumask *mask,
+				      struct notifier_block *nb);
+void cass_comp_irq_detach(struct cass_dev *hw, struct cass_irq *irq,
+			  struct notifier_block *nb);
+
+void cxi_register_hw_errors(struct cass_dev *hw,
+			    struct cxi_reg_err_flg *reg_err_flg);
+void cxi_unregister_hw_errors(struct cass_dev *hw,
+			      struct cxi_reg_err_flg *reg_err_flg);
+void cxi_enable_hw_errors(struct cass_dev *hw, unsigned int irq, bool is_ext,
+			  const unsigned long *bitmask);
+void cxi_disable_hw_errors(struct cass_dev *hw, unsigned int irq, bool is_ext,
+			   const unsigned long *bitmask);
+
+bool cass_phy_is_headshell_removed(struct cass_dev *hw);
+void cass_phy_set_state(enum cass_phy_state state, struct cass_dev *hw);
+
+void start_pcie_monitoring(struct cass_dev *hw);
+void stop_pcie_monitoring(struct cass_dev *hw);
+void cass_set_outstanding_limit(struct cass_dev *hw);
+
+void cass_disable_device(struct pci_dev *pdev, const char *reason);
+extern bool disable_on_error;
+void uncor_cb(struct cass_dev *hw, unsigned int irq, bool is_ext,
+	      unsigned int bitn);
+
+#endif	/* _CASS_CORE_H */

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2024 Hewlett Packard Enterprise Development LP */
+
+#include "cass_core.h"
+
+int cxi_dev_info_get(struct cxi_dev *dev,
+		     struct cxi_dev_info_use *devinfo)
+{
+	struct cass_dev *hw = container_of(dev, struct cass_dev, cdev);
+	int speed;
+	int state;
+
+	devinfo->nid = hw->cdev.prop.nid;
+	devinfo->pid_bits = hw->cdev.prop.pid_bits;
+	devinfo->pid_count = hw->cdev.prop.pid_count;
+	devinfo->pid_granule = hw->cdev.prop.pid_granule;
+	devinfo->min_free_shift = hw->cdev.prop.min_free_shift;
+	devinfo->rdzv_get_idx = hw->cdev.prop.rdzv_get_idx;
+	devinfo->device_rev = hw->cdev.prop.device_rev;
+	devinfo->device_proto = hw->cdev.prop.device_proto;
+	devinfo->device_platform = hw->cdev.prop.device_platform;
+	devinfo->vendor_id = hw->rev.vendor_id;
+	devinfo->device_id = hw->rev.device_id;
+	devinfo->pct_eq = hw->pct_eq_n;
+	devinfo->uc_nic = hw->uc_nic;
+	devinfo->link_mtu = C_MAX_HPC_MTU;
+	devinfo->num_ptes = hw->cdev.prop.rsrcs.ptes.max;
+	devinfo->num_txqs = hw->cdev.prop.rsrcs.txqs.max;
+	devinfo->num_tgqs = hw->cdev.prop.rsrcs.tgqs.max;
+	devinfo->num_eqs = hw->cdev.prop.rsrcs.eqs.max;
+	devinfo->num_cts = hw->cdev.prop.rsrcs.cts.max;
+	devinfo->num_acs = hw->cdev.prop.rsrcs.acs.max;
+	devinfo->num_tles = hw->cdev.prop.rsrcs.tles.max;
+	devinfo->num_les = hw->cdev.prop.rsrcs.les.max;
+	devinfo->cassini_version = hw->cdev.prop.cassini_version;
+	devinfo->system_type_identifier = hw->cdev.system_type_identifier;
+
+	if (hw->port)
+		cass_sbl_link_mode_to_speed(hw->port->lattr.bl.link_mode, &speed);
+	else
+		speed = 0;
+	devinfo->link_speed = speed;
+
+	state = hw->phy.state == CASS_PHY_RUNNING;
+	devinfo->link_state = state;
+
+	if (!hw->fru_info_valid)
+		uc_cmd_get_fru(hw);
+
+	if (hw->fru_info[PLDM_FRU_FIELD_DESCRIPTION])
+		strscpy(devinfo->fru_description, hw->fru_info[PLDM_FRU_FIELD_DESCRIPTION],
+			sizeof(devinfo->fru_description) - 1);
+	else
+		strscpy(devinfo->fru_description, "Not Available",
+			sizeof(devinfo->fru_description) - 1);
+	devinfo->fru_description[sizeof(devinfo->fru_description) - 1] = '\0';
+
+	return 0;
+}
+EXPORT_SYMBOL(cxi_dev_info_get);