[PATCH 22/23] net: hpe: ss1: add the Ethernet personality

@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cray Cassini ethernet driver
+ * © Copyright 2018-2020 Cray Inc
+ * © Copyright 2020-2021 Hewlett Packard Enterprise Development LP
+ */
+
+/* TODO:
+ *
+ * - address all the TODOs
+ * - a lot of error handling
+ * - support PCI device removal (see srcu usage in cxi_user_core.c)
+ * - find a way to cleanly pass the hash types. Modify ethtool?
+ * - add support for RSS contexts
+ */
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/pci.h>
+
+#include "cxi_eth.h"
+#include "cxi_eth_debugfs.h"
+
+unsigned int rss_indir_size = 64;
+module_param(rss_indir_size, int, 0444);
+MODULE_PARM_DESC(rss_indir_size, "Size of RSS indirection table");
+
+unsigned int max_rss_queues = 16;
+module_param(max_rss_queues, int, 0444);
+MODULE_PARM_DESC(max_rss_queues, "Maximum number of RSS queues (power of 2)");
+
+unsigned int max_tx_queues = 16;
+module_param(max_tx_queues, int, 0444);
+MODULE_PARM_DESC(max_tx_queues, "Maximum number of TX queues");
+
+unsigned int small_pkts_buf_size = CXI_ETH_1_MB;
+module_param(small_pkts_buf_size, uint, 0444);
+MODULE_PARM_DESC(small_pkts_buf_size, "Size of a small packets buffer (4 KiB to 1 GiB, 4 KiB aligned)");
+
+unsigned int small_pkts_buf_count = 4;
+module_param(small_pkts_buf_count, uint, 0444);
+MODULE_PARM_DESC(small_pkts_buf_count, "Number of small packets buffers (1 to 1024)");
+
+unsigned int large_pkts_buf_count = 1024;
+module_param(large_pkts_buf_count, uint, 0444);
+MODULE_PARM_DESC(large_pkts_buf_count, "Number of large packets buffers (1 to 10000)");
+
+unsigned int buffer_threshold = 256;
+module_param(buffer_threshold, uint, 0444);
+MODULE_PARM_DESC(buffer_threshold, "Ethernet packets with length greater than"
+		 " or equal to this threshold are large packets (0 to 16383)");
+
+unsigned int lpe_cdt_thresh_id = 2;
+module_param(lpe_cdt_thresh_id, uint, 0444);
+MODULE_PARM_DESC(lpe_cdt_thresh_id, "LPE append credit id to use (0 to 3)");
+
+static unsigned int rx_repost_retry_ms = 20;
+module_param(rx_repost_retry_ms, uint, 0444);
+MODULE_PARM_DESC(rx_repost_retry_ms,
+		 "Time to wait before retrying to repost an RX buffer, in ms");
+
+static struct dentry *cxieth_debug_dir;
+unsigned long rx_repost_retry_jiffies;
+
+static int idc_dma_threshold_set(const char *val, const struct kernel_param *kp)
+{
+	unsigned long num;
+	int ret;
+
+	if (!val)
+		return -EINVAL;
+
+	ret = kstrtoul(val, 0, &num);
+	if (ret || num > C_MAX_IDC_PAYLOAD_RES)
+		return -EINVAL;
+
+	*((unsigned int *)kp->arg) = num;
+
+	return 0;
+}
+
+static const struct kernel_param_ops idc_dma_threshold_ops = {
+	.set = idc_dma_threshold_set,
+	.get = param_get_uint,
+};
+
+unsigned int idc_dma_threshold = C_MAX_IDC_PAYLOAD_RES;
+module_param_cb(idc_dma_threshold, &idc_dma_threshold_ops,
+		&idc_dma_threshold, 0644);
+MODULE_PARM_DESC(idc_dma_threshold, "Ethernet packets up to this size will be"
+		 " sent inline. Larger packets will use DMA. (0 to 224)");
+
+/* List of devices registered with this client */
+static LIST_HEAD(dev_list);
+static DEFINE_MUTEX(dev_list_mutex);
+
+static const struct net_device_ops cxi_eth_netdev_ops = {
+	.ndo_open = cxi_eth_open,
+	.ndo_start_xmit = cxi_eth_start_xmit,
+	.ndo_stop = cxi_eth_close,
+	.ndo_set_mac_address = cxi_eth_mac_addr,
+	.ndo_set_rx_mode = cxi_eth_set_rx_mode,
+	.ndo_change_mtu = cxi_change_mtu,
+	.ndo_hwtstamp_get = cxi_eth_hwtstamp_get,
+	.ndo_hwtstamp_set = cxi_eth_hwtstamp_set,
+};
+
+/* Core is adding a new device */
+static int add_device(struct cxi_dev *cxi_dev)
+{
+	struct net_device *ndev;
+	struct cxi_eth *dev;
+	int rc;
+
+	/* Disable Ethernet devices on VF devices for now */
+	if (!cxi_dev->is_physfn)
+		return -ENODEV;
+
+	ndev = alloc_etherdev_mqs(sizeof(struct cxi_eth), max_tx_queues,
+				  max_rss_queues + 1);
+	if (!ndev) {
+		rc = -ENOMEM;
+		goto err;
+	}
+
+	cxi_set_ethernet_threshold(cxi_dev, buffer_threshold);
+	cxi_set_roce_rcv_seg(cxi_dev, false);
+
+	SET_NETDEV_DEV(ndev, &cxi_dev->pdev->dev);
+
+	dev = netdev_priv(ndev);
+
+	dev->txqs = kcalloc(max_tx_queues, sizeof(*dev->txqs), GFP_KERNEL);
+	if (!dev->txqs) {
+		rc = -ENOMEM;
+		goto err_free_ether;
+	}
+
+	dev->rxqs = kcalloc(max_rss_queues + 1, sizeof(*dev->rxqs), GFP_KERNEL);
+	if (!dev->rxqs) {
+		rc = -ENOMEM;
+		goto err_free_txq;
+	}
+
+	dev->ndev = ndev;
+	dev->cxi_dev = cxi_dev;
+	spin_lock_init(&dev->cq_tgt_req_lock);
+	dev->is_c2 = cassini_version(&dev->cxi_dev->prop, CASSINI_2);
+
+	ndev->netdev_ops = &cxi_eth_netdev_ops;
+	ndev->ethtool_ops = &cxi_eth_ethtool_ops;
+	ndev->watchdog_timeo = CXI_ETH_TX_TIMEOUT;
+
+	//ndev->irq = cxi_dev->pdev->irq;
+	ndev->mtu = ETH_DATA_LEN;
+	ndev->min_mtu = ETH_ZLEN;
+	ndev->max_mtu = CXI_ETH_MAX_MTU;
+	ndev->tx_queue_len = TX_QUEUE_LEN_DEFAULT;
+
+	cxi_eth_devinfo(cxi_dev, &dev->eth_info);
+
+	eth_hw_addr_set(ndev, dev->eth_info.default_mac_addr);
+
+	dev->ringparam.rx_max_pending = LARGE_PKTS_BUF_COUNT_MAX;
+	dev->ringparam.rx_mini_max_pending = num_small_packets(SMALL_PKTS_BUF_COUNT_MAX);
+	dev->ringparam.tx_max_pending = CXI_MAX_CQ_COUNT;
+
+	dev->ringparam.rx_pending = large_pkts_buf_count;
+	dev->ringparam.rx_mini_pending = num_small_packets(small_pkts_buf_count);
+	dev->ringparam.tx_pending = min_t(unsigned int, CXI_MAX_CQ_COUNT,
+					  2 * dev->ndev->tx_queue_len);
+
+	/* Used to set min_free to be equal or higher than buffer_threshold. */
+	dev->min_free = roundup_pow_of_two(buffer_threshold) >>
+		dev->eth_info.min_free_shift;
+
+	ndev->hw_features = NETIF_F_HW_CSUM | NETIF_F_GSO | NETIF_F_SG |
+		NETIF_F_GRO | NETIF_F_RXCSUM | NETIF_F_HIGHDMA |
+		NETIF_F_RXHASH;
+
+	ndev->features = ndev->hw_features;
+	ndev->vlan_features = ndev->hw_features;
+	ndev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;
+
+	rc = netif_set_real_num_queues(ndev,
+				       min_t(unsigned int, 8, max_tx_queues),
+				       min_t(unsigned int, 8, max_rss_queues));
+	if (rc)
+		goto err_free_rxq;
+
+	rc = register_netdev(ndev);
+	if (rc)
+		goto err_free_rxq;
+
+	/* If the SerDes are up, we must report the carrier as on here,
+	 * since we've already missed the link up event which was sent from
+	 * cxi-ss1
+	 */
+	if (cxi_is_link_up(cxi_dev)) {
+		/* Without the off->on transition, the ip link state will go to
+		 * UNKNOWN.
+		 */
+		netif_carrier_off(ndev);
+		netif_carrier_on(ndev);
+	} else {
+		netif_carrier_off(ndev);
+	}
+
+	device_debugfs_create(cxi_dev->name, dev, cxieth_debug_dir);
+
+	mutex_lock(&dev_list_mutex);
+	list_add_tail(&dev->dev_list, &dev_list);
+	mutex_unlock(&dev_list_mutex);
+
+	dev_info(&cxi_dev->pdev->dev, "%s[%s]: Added Ethernet device",
+		 cxi_dev->name, netdev_name(ndev));
+
+	return 0;
+
+err_free_rxq:
+	kfree(dev->rxqs);
+err_free_txq:
+	kfree(dev->txqs);
+err_free_ether:
+	free_netdev(ndev);
+err:
+	dev_err(&cxi_dev->pdev->dev,
+		"%s[]: Failure(%d): Ethernet device not added",
+		 cxi_dev->name, rc);
+	return rc;
+}
+
+/* From a given CXI device, find the ethernet device. Must be locked. */
+static struct cxi_eth *find_eth_device(const struct cxi_dev *cxi_dev)
+{
+	struct cxi_eth *dev;
+
+	list_for_each_entry_reverse(dev, &dev_list, dev_list) {
+		if (dev->cxi_dev == cxi_dev)
+			return dev;
+	}
+
+	return NULL;
+}
+
+static void remove_device(struct cxi_dev *cxi_dev)
+{
+	struct cxi_eth *dev;
+
+	/* Ethernet devices on VF devices are not added for now, so not removed */
+	if (!cxi_dev->is_physfn)
+		return;
+
+	mutex_lock(&dev_list_mutex);
+	dev = find_eth_device(cxi_dev);
+	if (dev)
+		list_del(&dev->dev_list);
+	mutex_unlock(&dev_list_mutex);
+
+	if (!dev) {
+		dev_info(&cxi_dev->pdev->dev,
+			 "%s[]: Failure(NOT FOUND): no Ethernet device to remove",
+			 cxi_dev->name);
+		return;
+	}
+
+	debugfs_remove(dev->debug);
+
+	unregister_netdev(dev->ndev);
+
+	kfree(dev->rxqs);
+	kfree(dev->txqs);
+	free_netdev(dev->ndev);
+
+	dev_info(&cxi_dev->pdev->dev, "%s[]: Removed Ethernet device",
+		 cxi_dev->name);
+}
+
+static void async_event(struct cxi_dev *cxi_dev, enum cxi_async_event event)
+{
+	struct cxi_eth *dev;
+
+	mutex_lock(&dev_list_mutex);
+	dev = find_eth_device(cxi_dev);
+	mutex_unlock(&dev_list_mutex);
+
+	if (!dev)
+		return;
+
+	netdev_dbg(dev->ndev, "got async event %d\n", event);
+
+	switch (event) {
+	case CXI_EVENT_LINK_UP:
+		netif_carrier_on(dev->ndev);
+		break;
+
+	case CXI_EVENT_LINK_DOWN:
+		netif_carrier_off(dev->ndev);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static int cxi_netdev_event(struct notifier_block *this, unsigned long event,
+			    void *ptr)
+{
+	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
+
+	if (ndev->netdev_ops->ndo_open == &cxi_eth_open &&
+	    event == NETDEV_CHANGENAME) {
+		struct cxi_eth *dev = netdev_priv(ndev);
+
+		cxi_set_eth_name(dev->cxi_dev, netdev_name(ndev));
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block cxi_netdev_notifier = {
+	.notifier_call = cxi_netdev_event,
+};
+
+static struct cxi_client cxiu_client = {
+	.add = add_device,
+	.remove = remove_device,
+	.async_event = async_event,
+};
+
+static int __init init(void)
+{
+	unsigned int new;
+	int ret;
+
+	if (rss_indir_size && (!is_power_of_2(rss_indir_size) ||
+			       rss_indir_size > CXI_ETH_MAX_INDIR_ENTRIES)) {
+		rss_indir_size = min_t(unsigned int,
+				       roundup_pow_of_two(rss_indir_size),
+				       CXI_ETH_MAX_INDIR_ENTRIES);
+		pr_info("Adjusting rss_indir_size to %d\n", rss_indir_size);
+	}
+
+	if (max_rss_queues > CXI_ETH_MAX_RSS_QUEUES ||
+	    !is_power_of_2(max_rss_queues)) {
+		max_rss_queues = min_t(unsigned int,
+				       roundup_pow_of_two(max_rss_queues),
+				       CXI_ETH_MAX_RSS_QUEUES);
+		pr_info("Adjusting max_rss_queues to %d\n", max_rss_queues);
+	}
+
+	if (max_tx_queues > CXI_ETH_MAX_TX_QUEUES) {
+		max_tx_queues = CXI_ETH_MAX_TX_QUEUES;
+		pr_info("Adjusting max_tx_queues to %d\n", max_tx_queues);
+	}
+
+	if (small_pkts_buf_size < PAGE_SIZE) {
+		small_pkts_buf_size = PAGE_SIZE;
+		pr_info("Adjusting small_pkts_buf_size to %u\n",
+			small_pkts_buf_size);
+	} else if (small_pkts_buf_size > CXI_ETH_1_GB) {
+		small_pkts_buf_size = CXI_ETH_1_GB;
+		pr_info("Adjusting small_pkts_buf_size to %u\n",
+			small_pkts_buf_size);
+	} else if ((small_pkts_buf_size % PAGE_SIZE) != 0) {
+		small_pkts_buf_size = roundup(small_pkts_buf_size, PAGE_SIZE);
+		pr_info("Adjusting small_pkts_buf_size to %u\n",
+			small_pkts_buf_size);
+	}
+
+	new = clamp(small_pkts_buf_count,
+		    SMALL_PKTS_BUF_COUNT_MIN, SMALL_PKTS_BUF_COUNT_MAX);
+	if (new != small_pkts_buf_count) {
+		small_pkts_buf_count = new;
+		pr_info("Adjusting small_pkts_buf_count to %u\n",
+			small_pkts_buf_count);
+	}
+
+	new = clamp(large_pkts_buf_count,
+		    LARGE_PKTS_BUF_COUNT_MIN, LARGE_PKTS_BUF_COUNT_MAX);
+	if (new != large_pkts_buf_count) {
+		large_pkts_buf_count = new;
+		pr_info("Adjusting large_pkts_buf_count to %u\n",
+			large_pkts_buf_count);
+	}
+
+	if (!buffer_threshold) {
+		pr_err("buffer_threshold must be nonzero\n");
+		return -EINVAL;
+	}
+	if (buffer_threshold > 16383) {
+		buffer_threshold = 16383;
+		pr_info("Adjusting buffer_threshold to %u\n", buffer_threshold);
+	}
+
+	if (lpe_cdt_thresh_id > 3) {
+		lpe_cdt_thresh_id = 0;
+		pr_info("Resetting lpe_cdt_thresh_id to 0\n");
+	}
+
+	if (rx_repost_retry_ms < 1 || rx_repost_retry_ms > 5000) {
+		rx_repost_retry_ms = 20;
+		pr_info("Resetting rx_repost_retry_ms to 20ms\n");
+	}
+	rx_repost_retry_jiffies = msecs_to_jiffies(rx_repost_retry_ms);
+
+	ret = register_netdevice_notifier(&cxi_netdev_notifier);
+	if (ret) {
+		pr_err("Failed to register netdevice_notifier\n");
+		return -EINVAL;
+	}
+
+	cxieth_debug_dir = device_eth_debugfs_init();
+
+	ret = cxi_register_client(&cxiu_client);
+	if (ret) {
+		pr_err("Couldn't register CXI ethernet driver\n");
+		goto out;
+	}
+
+	return 0;
+
+out:
+	debugfs_remove(cxieth_debug_dir);
+	unregister_netdevice_notifier(&cxi_netdev_notifier);
+
+	return ret;
+}
+
+static void __exit cleanup(void)
+{
+	cxi_unregister_client(&cxiu_client);
+	unregister_netdevice_notifier(&cxi_netdev_notifier);
+	debugfs_remove(cxieth_debug_dir);
+}
+
+module_init(init);
+module_exit(cleanup);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Cray eXascale Interconnect (CXI) Ethernet driver");
+MODULE_AUTHOR("Cray Inc.");

@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2018 Hewlett Packard Enterprise Development LP */
+
+/* cxi-eth debug fs*/
+#include <linux/netdevice.h>
+#include "cxi_eth.h"
+#include "cxi_eth_debugfs.h"
+
+static u64 bucket_times[NB_BUCKETS];
+
+static void dump_rx_queue(struct seq_file *s, const struct rx_queue *rx)
+{
+	const struct bucket *bucket;
+	int i;
+
+	seq_printf(s, "  EQ=%u, PtlTE=%u TGT PRIO CQ=%u\n",
+		   rx->eq->eqn, rx->pt->id, cxi_cq_get_cqn(rx->cq_tgt_prio));
+
+	seq_printf(s, "  Preferred NUMA Node: %u\n", rx->node);
+	seq_printf(s, "  Preferred CPU: %u\n", rx->cpu);
+
+	seq_printf(s, "  list rx_unallocated=%s\n",
+		   list_empty(&rx->rx_unallocated) ? "empty" : "not empty");
+	seq_printf(s, "  list rx_ready=%s\n",
+		   list_empty(&rx->rx_ready) ? "empty" : "not empty");
+	seq_printf(s, "  list rx_in_use=%s\n",
+		   list_empty(&rx->rx_in_use) ? "empty" : "not empty");
+
+	seq_printf(s, "  rx_bufs_count=%u\n", rx->rx_bufs_count);
+	seq_printf(s, "  append_prio=%llu\n", rx->stats.append_prio);
+	seq_printf(s, "  append_req=%llu\n", rx->stats.append_req);
+	seq_printf(s, "  append_failed=%llu\n", rx->stats.append_failed);
+	seq_printf(s, "  unlinked_prio=%llu\n", rx->stats.unlinked_prio);
+	seq_printf(s, "  unlinked_req=%llu\n", rx->stats.unlinked_req);
+	seq_printf(s, "  Last bad RC for dropped fragment: %u\n",
+		   rx->last_bad_frag_drop_rc);
+
+	/* Each bucket */
+	seq_puts(s, "  bucket for napi_schedule\n");
+	bucket = &rx->eth_napi_schedule;
+	if (bucket->min_ts == UINT_MAX)
+		seq_printf(s, "    min_ts=%llu\n", 0ULL);
+	else
+		seq_printf(s, "    min_ts=%llu\n", bucket->min_ts);
+	seq_printf(s, "    max_ts=%llu\n", bucket->max_ts);
+	for (i = 0; i < NB_BUCKETS; i++) {
+		if (i < (NB_BUCKETS - 1))
+			seq_printf(s, "      < %llu ns       %llu\n",
+				   bucket_times[i],
+				   bucket->b[i]);
+		else
+			seq_printf(s, "      higher          %llu\n",
+				   bucket->b[i]);
+	}
+	seq_puts(s, "\n");
+}
+
+static void dump_cq_info(struct seq_file *s, struct cxi_cq *cq)
+{
+	seq_printf(s, "    id: %u\n", cxi_cq_get_cqn(cq));
+	seq_printf(s, "    ptrs: size32=%u, rp32=%llu, hw_rp32=%u, wp32=%llu, hw_wp32=%llu\n",
+		   cq->size32, cq->rp32, cq->status->rd_ptr * 2,
+		   cq->wp32, cq->hw_wp32);
+	seq_printf(s, "    free CQ slots: %d\n", __cxi_cq_free_slots(cq));
+}
+
+static void dump_tx_queue(struct seq_file *s, const struct tx_queue *txq,
+			  struct cxi_eth *dev)
+{
+	struct netdev_queue *dev_queue =
+		netdev_get_tx_queue(txq->dev->ndev, txq->id);
+
+	seq_printf(s, "  EQ=%u\n", txq->eq->eqn);
+	seq_puts(s,   "  eth1 CQ:\n");
+	dump_cq_info(s, txq->eth1_cq);
+	if (dev->eth2_active) {
+		seq_puts(s,   "  eth2 CQ:\n");
+		dump_cq_info(s, txq->eth2_cq);
+	}
+	seq_puts(s,   "  shared CQ:\n");
+	dump_cq_info(s, txq->shared_cq);
+
+	seq_printf(s, "  Preferred NUMA Node: %u\n", txq->node);
+	seq_printf(s, "  Preferred CPU: %u\n", txq->cpu);
+	seq_printf(s, "  polling: %llu\n", txq->stats.polling);
+	seq_printf(s, "  TXQ EQ credits: %d\n", atomic_read(&txq->dma_eq_cdt));
+	seq_printf(s, "  TXQ stopped count for full CQ: %llu\n", txq->stats.stopped);
+	seq_printf(s, "  TXQ stopped for no EQ credits: %llu\n", txq->stats.stopped_eq);
+	seq_printf(s, "  TXQ restarted count: %llu\n", txq->stats.restarted);
+	seq_printf(s, "  TXQ got busy: %llu\n", txq->stats.tx_busy);
+	seq_printf(s, "  idc: %llu\n", txq->stats.idc);
+	seq_printf(s, "  idc_bytes: %llu\n", txq->stats.idc_bytes);
+	seq_printf(s, "  dma: %llu\n", txq->stats.dma);
+	seq_printf(s, "  dma_bytes: %llu\n", txq->stats.dma_bytes);
+	seq_printf(s, "  dma_forced: %llu\n", txq->stats.dma_forced);
+	seq_printf(s, "  force_dma: %u\n", txq->force_dma);
+	seq_printf(s, "  force_dma_interval: %u\n", txq->force_dma_interval);
+	seq_printf(s, "  netdev queue state: 0x%lx\n", dev_queue->state);
+}
+
+static int dump_dev(struct seq_file *s, void *unused)
+{
+	struct cxi_eth *dev = s->private;
+	u64 min_ts = UINT_MAX;
+	u64 max_ts = 0;
+	int i;
+	int j;
+
+	seq_printf(s, "rss_indir_size=%u\n", rss_indir_size);
+	seq_printf(s, "idc_dma_threshold=%u\n", idc_dma_threshold);
+	seq_printf(s, "small_pkts_buf_size=%u\n", small_pkts_buf_size);
+	seq_printf(s, "small_pkts_buf_count=%u\n", small_pkts_buf_count);
+	seq_printf(s, "large_pkts_buf_count=%u\n", large_pkts_buf_count);
+	seq_printf(s, "num rx_pending=%u\n", dev->ringparam.rx_pending);
+	seq_printf(s, "num rx_mini_pending=%u\n", dev->ringparam.rx_mini_pending);
+	seq_printf(s, "num tx_pending=%u\n", dev->ringparam.tx_pending);
+	seq_printf(s, "num_prio_buffers=%u\n", dev->num_prio_buffers);
+	seq_printf(s, "num_req_buffers=%u\n", dev->num_req_buffers);
+
+	if (!dev->is_active)
+		return 0;
+
+	seq_printf(s, "shared TGT REQ CQ=%u\n",
+		   cxi_cq_get_cqn(dev->cq_tgt_req));
+
+	for (i = 0; i < dev->res.rss_queues; i++) {
+		seq_printf(s, "RX queue %u\n", i);
+		dump_rx_queue(s, &dev->rxqs[i]);
+	}
+
+	if (dev->res.rss_queues) {
+		seq_printf(s, "PTP RX queue %u\n", PTP_RX_Q);
+		dump_rx_queue(s, &dev->rxqs[PTP_RX_Q]);
+	}
+
+	for (i = 0; i < dev->cur_txqs; i++) {
+		seq_printf(s, "TX queue %u\n", i);
+		dump_tx_queue(s, &dev->txqs[i], dev);
+	}
+
+	/* Sum all buckets */
+	seq_puts(s, "\n");
+	seq_puts(s, "ALL buckets for RX napi_schedule\n");
+
+	for (i = 0; i < dev->res.rss_queues; i++) {
+		const struct bucket *bucket = &dev->rxqs[i].eth_napi_schedule;
+
+		if (min_ts > bucket->min_ts)
+			min_ts = bucket->min_ts;
+		if (max_ts < bucket->max_ts)
+			max_ts = bucket->max_ts;
+	}
+
+	if (min_ts == UINT_MAX)
+		min_ts = 0;
+
+	seq_printf(s, "    min_ts=%llu\n", min_ts);
+	seq_printf(s, "    max_ts=%llu\n", max_ts);
+
+	for (i = 0; i < NB_BUCKETS; i++) {
+		u64 nb = 0;
+
+		for (j = 0; j < dev->res.rss_queues; j++)
+			nb += dev->rxqs[j].eth_napi_schedule.b[i];
+
+		if (i < (NB_BUCKETS - 1))
+			seq_printf(s, "      < %llu ns       %llu\n",
+				   bucket_times[i],
+				   nb);
+		else
+			seq_printf(s, "      higher          %llu\n",
+				   nb);
+	}
+	seq_puts(s, "\n");
+
+	return 0;
+}
+
+static int dump_dev_debug_dev_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, dump_dev, inode->i_private);
+}
+
+static const struct file_operations cxi_eth_fops = {
+	.owner = THIS_MODULE,
+	.open = dump_dev_debug_dev_open,
+	.read = seq_read,
+	.llseek  = seq_lseek,
+	.release = single_release,
+};
+
+struct dentry *device_eth_debugfs_init(void)
+{
+	int i;
+
+	for (i = 0; i < NB_BUCKETS; i++)
+		bucket_times[i] = 1 << i;
+	return debugfs_create_dir("cxi_eth", NULL);
+}
+
+void device_debugfs_create(char *name, struct cxi_eth *dev,
+			   struct dentry *cxieth_debug_dir)
+{
+	dev->debug = debugfs_create_file(name, 0444, cxieth_debug_dir,
+					 dev, &cxi_eth_fops);
+}

@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cray Cassini ethernet driver - hardware timestamp configuration.
+ * © Copyright 2021 Hewlett Packard Enterprise Development LP
+ */
+
+#include <linux/netdevice.h>
+#include <linux/net_tstamp.h>
+
+#include "cxi_eth.h"
+
+int cxi_eth_hwtstamp_get(struct net_device *ndev,
+			 struct kernel_hwtstamp_config *config)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	hwtstamp_config_to_kernel(config, &dev->tstamp_config);
+
+	return 0;
+}
+
+int cxi_eth_hwtstamp_set(struct net_device *ndev,
+			 struct kernel_hwtstamp_config *config,
+			 struct netlink_ext_ack *extack)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct hwtstamp_config cfg;
+	int rc;
+
+	hwtstamp_config_from_kernel(&cfg, config);
+
+	rc = cxi_eth_cfg_timestamp(dev->cxi_dev, &cfg);
+	if (rc)
+		return rc;
+
+	dev->tstamp_config = cfg;
+
+	dev->ptp_ts_enabled = cfg.rx_filter == HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
+
+	/* Report the configuration actually applied, e.g. TX forced on. */
+	hwtstamp_config_to_kernel(config, &cfg);
+
+	return 0;
+}

@@ -0,0 +1,2367 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cray Cassini ethernet driver
+ * © Copyright 2018-2020 Cray Inc
+ * © Copyright 2020 Hewlett Packard Enterprise Development LP
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/netdevice.h>
+#include <linux/pci.h>
+#include <linux/tcp.h>
+#include <linux/udp.h>
+#include <linux/vmalloc.h>
+#include <net/ipv6.h>
+
+#include "cxi_eth.h"
+#include "cxi_core.h"
+
+#define ROCE_V2_UDP_DPORT	4791
+
+static unsigned int tx_eq_count;
+module_param(tx_eq_count, uint, 0444);
+MODULE_PARM_DESC(tx_eq_count, "Override the number of entries in transmit event queue");
+
+static int cxi_rx_eth_poll(struct napi_struct *napi, int budget);
+static void rx_eq_cb(void *context);
+
+static void err_eq_cb(void *context)
+{
+	struct cxi_eth *dev = context;
+	const union c_event *event;
+
+	while ((event = cxi_eq_get_event(dev->err_eq))) {
+		switch (event->hdr.event_type) {
+		case C_EVENT_COMMAND_FAILURE:
+			netdev_warn(dev->ndev, "%s CQ %u disabled\n",
+				    event->cmd_fail.is_target ?
+				    "target" : "transmit",
+				    event->cmd_fail.cq_id);
+			break;
+		default:
+			netdev_warn(dev->ndev, "Unhandled event %u, rc %d\n",
+				    event->hdr.event_type,
+				    event->hdr.return_code);
+			print_hex_dump(KERN_INFO,
+				       "cxi eth unexpected tx event: ",
+				       DUMP_PREFIX_ADDRESS, 16, 1, event,
+				       (8 << event->hdr.event_size), false);
+		}
+
+		cxi_eq_ack_events(dev->err_eq);
+	}
+
+	cxi_eq_int_enable(dev->err_eq);
+}
+
+static void cancel_tx_tstamp(struct cxi_eth *dev, struct sk_buff *skb)
+{
+	if (cmpxchg(&dev->tstamp_skb, skb, NULL) == skb)
+		skb_shinfo(skb)->tx_flags &= ~SKBTX_IN_PROGRESS;
+}
+
+static void finalize_tx_tstamp(struct cxi_eth *dev, struct sk_buff *skb)
+{
+	struct skb_shared_hwtstamps tstamps;
+	int rc;
+
+	if (skb != dev->tstamp_skb) {
+		netdev_warn_once(dev->ndev, "skb != dev->tstamp_skb : %p != %p",
+				 skb, dev->tstamp_skb);
+		return;
+	}
+
+	rc = cxi_eth_get_tx_timestamp(dev->cxi_dev, &tstamps);
+	if (!rc)
+		skb_tstamp_tx(skb, &tstamps);
+
+	/* Release timestamp slot */
+	cmpxchg(&dev->tstamp_skb, skb, NULL);
+}
+
+static void unmap_frags(struct device *dma_dev, struct sk_buff *skb)
+{
+	struct cb_data *cb = (struct cb_data *)skb->cb;
+	int i;
+
+	for (i = 0; i < cb->num_frags; i++) {
+		int len;
+
+		if (i < MAX_CB_LEN) {
+			len = cb->dma_len[i];
+		} else {
+			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
+
+			len = skb_frag_size(frag);
+		}
+
+		dma_unmap_single(dma_dev, cb->dma_addr[i], len, DMA_TO_DEVICE);
+	}
+}
+
+/* Prepare the segments in the command, mapping them for DMA */
+static int prepare_frags(struct device *dma_dev, struct sk_buff *skb,
+			 struct c_dma_eth_cmd *dma_eth)
+{
+	struct cb_data *cb = (struct cb_data *)skb->cb;
+	int i;
+
+	BUILD_BUG_ON(sizeof(struct cb_data) > sizeof(skb->cb));
+
+	dma_eth->num_segments = 1 + skb_shinfo(skb)->nr_frags;
+
+	if (skb_shinfo(skb)->nr_frags) {
+		unsigned int segnum = 0;
+
+		cb->dma_len[segnum] = dma_eth->len[segnum] = skb_headlen(skb);
+		segnum++;
+
+		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+			dma_eth->len[segnum] = skb_frag_size(frag);
+
+			if (segnum < MAX_CB_LEN)
+				cb->dma_len[segnum] = dma_eth->len[segnum];
+
+			segnum++;
+		}
+	} else {
+		cb->dma_len[0] = skb->len;
+		dma_eth->len[0] = skb->len;
+	}
+
+	cb->num_frags = 0;
+
+	for (i = 0; i < dma_eth->num_segments; i++) {
+		dma_addr_t dma_addr;
+
+		if (i == 0) {
+			dma_addr = dma_map_single(dma_dev, skb->data,
+						  dma_eth->len[i], DMA_TO_DEVICE);
+		} else {
+			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
+
+			dma_addr = dma_map_single(dma_dev, skb_frag_address(frag),
+						  dma_eth->len[i], DMA_TO_DEVICE);
+		}
+
+		if (dma_mapping_error(dma_dev, dma_addr)) {
+			unmap_frags(dma_dev, skb);
+			return -ENOMEM;
+		}
+
+		cb->dma_addr[i] = dma_eth->addr[i] = dma_addr;
+		cb->num_frags++;
+	}
+
+	return 0;
+}
+
+/* A packet was sent with a DMA command. Returns the number of EQ
+ * credits to return to the TX queue.
+ */
+static unsigned int eth_complete(struct cxi_eth *dev, struct tx_queue *txq,
+				 const union c_event *event, int budget,
+				 unsigned int *pkts, unsigned int *bytes)
+{
+	struct net_device *ndev = dev->ndev;
+	struct sk_buff *skb;
+	enum c_return_code return_code;
+
+	if (event->hdr.event_size == C_EVENT_SIZE_16_BYTE) {
+		return_code = event->init_short.return_code;
+		skb = (void *)event->init_short.user_ptr;
+	} else {
+		return_code = event->init_long.return_code;
+		skb = (void *)event->init_long.user_ptr;
+	}
+
+	if (skb == NULL) {
+		/* Bad IDC commands will generate a SEND event on error. */
+		if (return_code == C_RC_OK)
+			netdev_warn_once(dev->ndev,
+					 "Successful command with NULL skb\n");
+
+		ndev->stats.tx_dropped++;
+
+		return 0;
+	}
+
+	if (return_code == C_RC_OK) {
+		ndev->stats.tx_packets++;
+		ndev->stats.tx_bytes += skb->len;
+		*pkts += 1;
+		*bytes += skb->len;
+	} else {
+		netdev_err_once(dev->ndev,
+				"DMA command failed to complete: %d\n",
+				return_code);
+		ndev->stats.tx_dropped++;
+	}
+
+	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)
+		finalize_tx_tstamp(dev, skb);
+
+	unmap_frags(&dev->cxi_dev->pdev->dev, skb);
+
+	napi_consume_skb(skb, budget);
+
+	return 1;
+}
+
+static int cxi_tx_eth_poll(struct napi_struct *napi, int budget)
+{
+	struct tx_queue *txq = container_of(napi, struct tx_queue, napi);
+	struct cxi_eth *dev = txq->dev;
+	struct netdev_queue *dev_queue =
+		netdev_get_tx_queue(dev->ndev, txq->id);
+	const union c_event *event;
+	int work_done = 0;
+	unsigned int event_count = 0;
+	unsigned int pkts = 0;
+	unsigned int bytes = 0;
+	unsigned int cdts = 0;
+
+	txq->stats.polling++;
+
+	while (work_done < budget && (event = cxi_eq_get_event(txq->eq))) {
+		switch (event->hdr.event_type) {
+		case C_EVENT_SEND:
+			cdts += eth_complete(dev, txq, event, budget, &pkts, &bytes);
+			work_done++;
+			break;
+		default:
+			netdev_warn(dev->ndev, "Unhandled event %u, rc %d\n",
+				    event->hdr.event_type,
+				    event->hdr.return_code);
+			print_hex_dump(KERN_INFO,
+				       "cxi eth unexpected tx event: ",
+				       DUMP_PREFIX_ADDRESS, 16, 1, event,
+				       (8 << event->hdr.event_size), false);
+		}
+
+		event_count++;
+		if (event_count == 16) {
+			cxi_eq_int_disable(txq->eq);
+			event_count = 0;
+
+			atomic_add(cdts, &txq->dma_eq_cdt);
+			cdts = 0;
+		}
+	}
+
+	BUG_ON(work_done > budget);
+
+	/* Check to see if the TX queue needs to be re-enabled. */
+	if (netif_tx_queue_stopped(dev_queue)) {
+		if (__cxi_cq_free_slots(txq->shared_cq) >= 16 &&
+		    __cxi_cq_free_slots(txq->eth1_cq) >= 16 &&
+		    (!dev->eth2_active || __cxi_cq_free_slots(txq->eth2_cq) >= 16) &&
+		    atomic_read(&txq->dma_eq_cdt) > 0) {
+			txq->stats.restarted++;
+			netif_tx_wake_queue(dev_queue);
+		}
+	}
+
+	if (pkts)
+		netdev_tx_completed_queue(dev_queue, pkts, bytes);
+
+	if (work_done < budget && napi_complete_done(napi, work_done)) {
+		/* Ack events and re-enable the EQ interrupt. */
+		cxi_eq_int_enable(txq->eq);
+	} else if (event_count) {
+		cxi_eq_int_disable(txq->eq);
+	}
+
+	atomic_add(cdts, &txq->dma_eq_cdt);
+
+	return work_done;
+}
+
+static void tx_eq_cb(void *context)
+{
+	struct tx_queue *tx = context;
+
+	napi_schedule(&tx->napi);
+}
+
+void enable_tx_queue(struct tx_queue *tx)
+{
+	napi_enable(&tx->napi);
+	cxi_eq_int_enable(tx->eq);
+}
+
+void disable_tx_queue(struct tx_queue *tx)
+{
+	cxi_eq_int_disable(tx->eq);
+	napi_disable(&tx->napi);
+}
+
+void free_tx_queue(struct tx_queue *tx)
+{
+	netif_napi_del(&tx->napi);
+
+	tx->untagged_cq = NULL;
+
+	cxi_cq_free(tx->shared_cq);
+	tx->shared_cq = NULL;
+
+	cxi_cq_free(tx->eth1_cq);
+	tx->eth1_cq = NULL;
+
+	if (tx->eth2_cq) {
+		cxi_cq_free(tx->eth2_cq);
+		tx->eth2_cq = NULL;
+	}
+
+	cxi_eq_free(tx->eq);
+	tx->eq = NULL;
+
+	if (is_vmalloc_addr(tx->eq_attr.queue))
+		cxi_unmap(tx->eq_md);
+	kvfree(tx->eq_attr.queue);
+	tx->eq_attr.queue = NULL;
+
+	tx->dev = NULL;
+}
+
+static void cxi_eth_get_node_cpu(struct cxi_eth *dev, unsigned int queue_id,
+				 unsigned int *queue_node,
+				 unsigned int *queue_cpu)
+{
+	int node;
+	int cpu;
+	int cpu_offset;
+	const struct cpumask *cpus;
+	int cpu_count;
+	int num_cpu_nodes = 0;
+	int i;
+
+	/* Figure out how many numa nodes are online and contain
+	 * CPUs. Apparently linux will reorder the nodes so that the
+	 * empty ones are at the end of the list. If it's not the
+	 * case, then further changes will be needed.
+	 */
+	for_each_online_node(node) {
+		cpus = cpumask_of_node(node);
+		if (!cpumask_empty(cpus))
+			num_cpu_nodes++;
+	}
+	if (!num_cpu_nodes || dev_to_node(&dev->cxi_dev->pdev->dev) < 0)
+		goto fallback;
+
+	/* Round robin queues across all the nodes in the system instead of
+	 * overloading 1 node with multiple queues. Favor the local node to
+	 * start with.
+	 */
+	node = (dev_to_node(&dev->cxi_dev->pdev->dev) + queue_id) % num_cpu_nodes;
+	if (!node_online(node)) {
+		/* For sanity, but that shouldn't be possible. */
+		netdev_err(dev->ndev, "Node %u not online. Defaulting to %u\n",
+			   node, dev_to_node(&dev->cxi_dev->pdev->dev));
+		node = dev_to_node(&dev->cxi_dev->pdev->dev);
+	}
+
+	/* If the queue ID exceeds the number of nodes, multiple queues will be
+	 * mapped to the same node. For this case, round robin queues within the
+	 * node.
+	 */
+	cpus = cpumask_of_node(node);
+	if (cpumask_empty(cpus)) {
+		/* For sanity, but that shouldn't be possible. */
+		netdev_err(dev->ndev, "Node %u is empty. Defaulting to all cpus.\n",
+			   node);
+		cpus = cpu_all_mask;
+	}
+
+	cpu_count = 0;
+	for_each_cpu(cpu, cpus) {
+		if (cpu_is_offline(cpu))
+			continue;
+		/* We only care about CPUs which belong to the NUMA node. */
+		if (cpumask_test_cpu(cpu, cpus))
+			cpu_count++;
+	}
+
+	if (!cpu_count)
+		goto fallback;
+	cpu_offset = (queue_id / num_cpu_nodes) % cpu_count;
+	i = 0;
+	for_each_cpu(cpu, cpus) {
+		if (cpu_is_offline(cpu))
+			continue;
+		if (cpumask_test_cpu(cpu, cpus)) {
+			if (i == cpu_offset)
+				break;
+			i++;
+		}
+	}
+
+	if (cpu < nr_cpu_ids)
+		goto out;
+
+fallback:
+	cpu = cpumask_local_spread(queue_id,
+				   dev_to_node(&dev->cxi_dev->pdev->dev));
+out:
+	*queue_node = cpu_to_node(cpu);
+	*queue_cpu = cpu;
+}
+
+int alloc_tx_queue(struct cxi_eth *dev, unsigned int id)
+{
+	struct tx_queue *tx = &dev->txqs[id];
+	struct cxi_cq_alloc_opts cq_alloc_opts = {};
+	struct c_cstate_cmd c_state = {};
+	unsigned int eq_count;
+	int rc;
+
+	memset(tx, 0, sizeof(*tx));
+
+	cxi_eth_get_node_cpu(dev, id, &tx->node, &tx->cpu);
+
+	tx->force_dma_interval = dev->ndev->tx_queue_len / 8;
+	tx->force_dma = tx->force_dma_interval;
+
+	cq_alloc_opts.count = min_t(unsigned int, CXI_MAX_CQ_COUNT,
+				    2 * dev->ndev->tx_queue_len);
+
+	eq_count = tx_eq_count ? tx_eq_count : 4 * cq_alloc_opts.count;
+	tx->eq_attr.queue_len = PAGE_ALIGN(C_EE_CFG_ECB_SIZE * eq_count);
+
+	tx->eq_attr.queue = kvzalloc_node(tx->eq_attr.queue_len, GFP_KERNEL,
+					  tx->node);
+	if (!tx->eq_attr.queue) {
+		rc = -ENOMEM;
+		goto err;
+	}
+
+	/* Keep a 5% margin for IDC send errors if they ever happen. */
+	atomic_set(&tx->dma_eq_cdt, (eq_count * 95) / 100);
+
+	if (is_vmalloc_addr(tx->eq_attr.queue)) {
+		tx->eq_md = cxi_map(dev->lni, (uintptr_t)tx->eq_attr.queue,
+				    tx->eq_attr.queue_len, CXI_MAP_WRITE, NULL);
+		if (IS_ERR(tx->eq_md)) {
+			rc = PTR_ERR(tx->eq_md);
+			goto err_free_eq_buf;
+		}
+	} else {
+		tx->eq_attr.flags = CXI_EQ_PASSTHROUGH;
+	}
+
+	tx->eq_attr.cpu_affinity = tx->cpu;
+	tx->eq = cxi_eq_alloc(dev->lni, tx->eq_md, &tx->eq_attr, tx_eq_cb, tx,
+			      NULL, NULL);
+	if (IS_ERR(tx->eq)) {
+		rc = PTR_ERR(tx->eq);
+		goto err_unmap_eq_buf;
+	}
+
+	cq_alloc_opts.flags = CXI_CQ_IS_TX | CXI_CQ_TX_ETHERNET;
+	cq_alloc_opts.lcid = dev->eth1_cp->lcid;
+	tx->eth1_cq = cxi_cq_alloc(dev->lni, dev->err_eq, &cq_alloc_opts,
+				   tx->node);
+	if (IS_ERR(tx->eth1_cq)) {
+		rc = PTR_ERR(tx->eth1_cq);
+		goto err_free_eq;
+	}
+
+	cq_alloc_opts.lcid = dev->shared_cp->lcid;
+	tx->shared_cq = cxi_cq_alloc(dev->lni, dev->err_eq, &cq_alloc_opts,
+				     tx->node);
+	if (IS_ERR(tx->shared_cq)) {
+		rc = PTR_ERR(tx->shared_cq);
+		goto err_free_eth1_cq;
+	}
+
+	if (dev->eth2_active) {
+		cq_alloc_opts.lcid = dev->eth2_cp->lcid;
+		tx->eth2_cq = cxi_cq_alloc(dev->lni, dev->err_eq,
+					   &cq_alloc_opts, tx->node);
+
+		if (IS_ERR(tx->eth2_cq)) {
+			rc = PTR_ERR(tx->eth2_cq);
+			goto err_free_shared_cq;
+		}
+	}
+
+	/* Add C_STATE command. */
+	c_state.eq = tx->eq->eqn;
+	c_state.restricted = 1;
+	c_state.event_success_disable = 1;
+	c_state.event_send_disable = 1;
+
+	rc = cxi_cq_emit_c_state(tx->eth1_cq, &c_state);
+	if (rc) {
+		netdev_err(dev->ndev, "Command emit failed: %d\n", rc);
+		goto err_free_eth2_cq;
+	}
+
+	cxi_cq_ring(tx->eth1_cq);
+
+	rc = cxi_cq_emit_c_state(tx->shared_cq, &c_state);
+	if (rc) {
+		netdev_err(dev->ndev, "Command emit failed: %d\n", rc);
+		goto err_free_eth2_cq;
+	}
+
+	cxi_cq_ring(tx->shared_cq);
+
+	if (dev->eth2_active) {
+		rc = cxi_cq_emit_c_state(tx->eth2_cq, &c_state);
+		if (rc) {
+			netdev_err(dev->ndev, "Command emit failed: %d\n", rc);
+			goto err_free_eth2_cq;
+		}
+
+		cxi_cq_ring(tx->eth2_cq);
+	}
+
+	/* Determine which tx_queue CQ should be used for untagged traffic */
+	if (dev->cxi_dev->untagged_eth_pcp == dev->eth1_pcp)
+		tx->untagged_cq = tx->eth1_cq;
+	else if (dev->eth2_active &&
+		 dev->cxi_dev->untagged_eth_pcp == dev->eth2_pcp)
+		tx->untagged_cq = tx->eth2_cq;
+	else
+		tx->untagged_cq = tx->shared_cq;
+
+	tx->dev = dev;
+	tx->id = id;
+
+	NETIF_NAPI_ADD_TX(dev->ndev, &tx->napi, cxi_tx_eth_poll);
+
+	return 0;
+
+err_free_eth2_cq:
+	if (dev->eth2_active) {
+		cxi_cq_free(tx->eth2_cq);
+		tx->eth2_cq = NULL;
+	}
+err_free_shared_cq:
+	cxi_cq_free(tx->shared_cq);
+	tx->shared_cq = NULL;
+err_free_eth1_cq:
+	cxi_cq_free(tx->eth1_cq);
+	tx->eth1_cq = NULL;
+err_free_eq:
+	cxi_eq_free(tx->eq);
+	tx->eq = NULL;
+err_unmap_eq_buf:
+	if (is_vmalloc_addr(tx->eq_attr.queue))
+		cxi_unmap(tx->eq_md);
+err_free_eq_buf:
+	kvfree(tx->eq_attr.queue);
+	tx->eq_attr.queue = NULL;
+err:
+	return rc;
+}
+
+/* Free the receive buffers on the ready lists.
+ * TODO: when closing the device, make sure all buffers are back on
+ * the ready lists.
+ */
+static void free_rx_buffers(struct rx_queue *rx)
+{
+	struct cxi_eth *dev = rx->dev;
+	struct rx_buffer *buf;
+
+	/* Move all buffers to the ready list and free them. */
+	list_splice_init(&rx->rx_in_use, &rx->rx_ready);
+
+	while ((buf = list_first_entry_or_null(&rx->rx_ready,
+					       struct rx_buffer, buf_list))) {
+		list_del(&buf->buf_list);
+		list_add_tail(&buf->buf_list, &rx->rx_unallocated);
+
+		if (buf->ptl_list == C_PTL_LIST_PRIORITY) {
+			if (buf->page) {
+				dma_unmap_page(&dev->cxi_dev->pdev->dev,
+					       dma_unmap_addr(buf, mapping),
+				PAGE_SIZE, DMA_FROM_DEVICE);
+
+				for (; buf->posted_count > 0; buf->posted_count--)
+					put_page(buf->page);
+
+				__free_page(buf->page);
+				buf->page = NULL;
+			}
+		} else {
+			cxi_unmap(buf->md);
+			kvfree(buf->data);
+		}
+		buf->data = NULL;
+	}
+}
+
+/* Allocate and map the receive buffers. */
+static int alloc_rx_buffers(struct rx_queue *rx, unsigned int count,
+			    enum c_ptl_list ptl_list)
+{
+	int rc;
+	unsigned int i;
+	struct rx_buffer *buf;
+	unsigned int data_size;
+	struct cxi_eth *dev = rx->dev;
+	int numa_node = dev_to_node(&dev->cxi_dev->pdev->dev);
+
+	if (ptl_list == C_PTL_LIST_PRIORITY)
+		data_size = dev->eth_info.max_segment_size;
+	else
+		data_size = small_pkts_buf_size;
+
+	for (i = 0; i < count; i++) {
+		buf = list_first_entry_or_null(&rx->rx_unallocated,
+					       struct rx_buffer, buf_list);
+		if (!buf)
+			break;
+
+		list_del(&buf->buf_list);
+
+		buf->data_size = data_size;
+
+		if (ptl_list == C_PTL_LIST_REQUEST) {
+			buf->data = kvzalloc_node(buf->data_size, GFP_KERNEL,
+						  numa_node);
+			if (!buf->data)
+				goto out_requeue;
+
+			buf->md = cxi_map(dev->lni, (uintptr_t)buf->data,
+					  buf->data_size, CXI_MAP_WRITE, NULL);
+			if (IS_ERR(buf->md)) {
+				rc = PTR_ERR(buf->md);
+				netdev_err(dev->ndev,
+					   "cxi_map failed: %d\n", rc);
+				kvfree(buf->data);
+				buf->data = NULL;
+				goto out_requeue;
+			}
+		}
+
+		/* Allocated. Enqueue on the corresponding ready list. */
+		buf->ptl_list = ptl_list;
+
+		list_add_tail(&buf->buf_list, &rx->rx_ready);
+
+		if (ptl_list == C_PTL_LIST_PRIORITY)
+			dev->num_prio_buffers++;
+		else
+			dev->num_req_buffers++;
+	}
+
+	return i;
+
+out_requeue:
+	list_add_tail(&buf->buf_list, &rx->rx_unallocated);
+
+	return i;
+}
+
+/* Re-post an RX buffer. The buffer is on the rx_ready list. */
+static int append_rx_buffer(struct rx_queue *rx, struct rx_buffer *buf,
+			    gfp_t gfp)
+{
+	int rc;
+	dma_addr_t mapping;
+	struct page *page;
+	struct cxi_eth *dev = rx->dev;
+	struct c_target_cmd cmd = {
+		.command.opcode = C_CMD_TGT_APPEND,
+		.ptl_list = buf->ptl_list,
+		.ptlte_index = rx->pt->id,
+		.op_put = 1,
+		.no_truncate = 1,
+		.event_link_disable = 1,
+		.length = buf->data_size,
+		.buffer_id = buf->id,
+		.unrestricted_end_ro = 1,
+		.unrestricted_body_ro = 1,
+		.event_unlink_disable = 1,
+	};
+	int i;
+	size_t posted_offset;
+
+	if (buf->ptl_list == C_PTL_LIST_PRIORITY) {
+		cmd.use_once = 1;
+
+		if (buf->page == NULL) {
+			page = alloc_page(gfp);
+			if (!page) {
+				rc = -ENOMEM;
+				goto fail;
+			}
+
+			mapping = dma_map_page(&dev->cxi_dev->pdev->dev, page,
+					       0, PAGE_SIZE,
+					       DMA_FROM_DEVICE);
+			if (dma_mapping_error(&dev->cxi_dev->pdev->dev,
+					      mapping)) {
+				__free_page(page);
+				rc = -EIO;
+				goto fail;
+			}
+			buf->page = page;
+			buf->mapping = mapping;
+		}
+
+		cmd.start = buf->mapping;
+		cmd.lac = dev->phys_lac;
+		buf->cur_offset = 0;
+		buf->posted_count = 0;
+		posted_offset = 0;
+
+		for (i = 0; i < rx->page_chunks; i++) {
+			cmd.start = buf->mapping + posted_offset;
+
+			rc = cxi_cq_emit_target(rx->cq_tgt_prio, &cmd);
+			if (rc)
+				goto fail;
+
+			get_page(buf->page);
+			buf->posted_count++;
+			posted_offset += dev->eth_info.max_segment_size;
+			rx->stats.append_prio++;
+		}
+	} else {
+		cmd.manage_local = 1;
+		cmd.min_free = dev->min_free;
+		cmd.start = buf->md->iova;
+		cmd.lac = buf->md->lac;
+
+		spin_lock_bh(&dev->cq_tgt_req_lock);
+		rc = cxi_cq_emit_target(dev->cq_tgt_req, &cmd);
+		spin_unlock_bh(&dev->cq_tgt_req_lock);
+
+		rx->stats.append_req++;
+	}
+
+fail:
+	if (rc) {
+		netdev_err_once(dev->ndev, "Append failed on list %d: %d\n",
+				buf->ptl_list, rc);
+		rx->stats.append_failed++;
+	} else {
+		list_del(&buf->buf_list);
+		list_add_tail(&buf->buf_list, &rx->rx_in_use);
+	}
+
+	return rc;
+}
+
+/* Append all ready RX buffers to the portal.
+ * Return the number of buffers appended.
+ */
+int post_rx_buffers(struct rx_queue *rx, gfp_t gfp)
+{
+	struct cxi_eth *dev = rx->dev;
+	unsigned int posted = 0;
+	unsigned int count = 0;
+	struct rx_buffer *buf;
+	struct rx_buffer *tmp;
+	int rc = 0;
+
+	list_for_each_entry_safe(buf, tmp, &rx->rx_ready, buf_list) {
+		rc = append_rx_buffer(rx, buf, gfp);
+		if (rc)
+			break;
+
+		/* Ring the CQ regularly */
+		if (buf->ptl_list == C_PTL_LIST_PRIORITY) {
+			posted++;
+			if (posted == 16) {
+				cxi_cq_ring(rx->cq_tgt_prio);
+				posted = 0;
+			}
+		} else {
+			spin_lock_bh(&dev->cq_tgt_req_lock);
+			cxi_cq_ring(dev->cq_tgt_req);
+			spin_unlock_bh(&dev->cq_tgt_req_lock);
+		}
+
+		count++;
+	}
+
+	if (posted)
+		cxi_cq_ring(rx->cq_tgt_prio);
+
+	return rc ? rc : count;
+}
+
+#define EXPECTED_TARGET_EVENT_SIZE (sizeof(struct c_event_target_enet))
+#define MIN_PACKET_SIZE 64U
+size_t get_rxq_eq_buf_size(struct cxi_eth *dev)
+{
+	size_t num_events;
+
+	/* Calculate the maximum number of events which could occur based on
+	 * RX buffer configuration.
+	 */
+	num_events = dev->ringparam.rx_pending +
+		(small_pkts_buf_count * (small_pkts_buf_size / MIN_PACKET_SIZE));
+
+	return ALIGN((num_events * EXPECTED_TARGET_EVENT_SIZE) +
+		     C_EE_CFG_ECB_SIZE, PAGE_SIZE);
+}
+
+/* CXI_CQ_UPDATE_HIGH_FREQ_EMPTY updates the read pointer every 16 slots.
+ * Since a read pointer status update could be missed on CQ wrap, 2 * 16 slots
+ * should be included as overhead.
+ *
+ * In addition, 4 event slots are reserved for status updates. Thus, an
+ * additional 4 slots should be included as overhead.
+ */
+#define TGT_CQ_OVERHEAD_COUNT ((2U * 16U) + 4U)
+
+int alloc_rx_queue(struct cxi_eth *dev, unsigned int id)
+{
+	int i;
+	int rc;
+	static const struct cxi_pt_alloc_opts pt_alloc_opts = {
+		.en_flowctrl = 1, /* TODO: needed for ethernet? */
+		.ethernet = 1,
+		.do_space_check = 1,
+		.lossless = 1,
+	};
+	struct cxi_eq_attr eq_attr = {};
+	struct cxi_cq_alloc_opts cq_alloc_opts = {};
+	struct rx_queue *rx = &dev->rxqs[id];
+	unsigned int prio_bufs;
+
+	memset(rx, 0, sizeof(*rx));
+
+	cxi_eth_get_node_cpu(dev, id, &rx->node, &rx->cpu);
+
+	rx->dev = dev;
+	rx->id = id;
+	rx->last_bad_frag_drop_rc = C_RC_OK;
+	rx->eth_napi_schedule.min_ts = UINT_MAX;
+
+	INIT_LIST_HEAD(&rx->rx_unallocated);
+	INIT_LIST_HEAD(&rx->rx_ready);
+	INIT_LIST_HEAD(&rx->rx_in_use);
+
+	/* Packets landing on priority buffers will be chopped into
+	 * max_segment_size pieces. Since pages are priority buffers, to avoid
+	 * wasting page memory, pages should be chunked in max_segment_size
+	 * pieces and posted.
+	 */
+	rx->page_chunks = PAGE_SIZE / dev->eth_info.max_segment_size;
+
+	/* The number of priority buffers, not pages, is the size of the RX
+	 * ring divided by number of page chunks. If the
+	 * PAGE_SIZE ==  max_segment_size, the number of prio_bufs will equal
+	 * rx_pending.
+	 */
+	prio_bufs = DIV_ROUND_UP(dev->ringparam.rx_pending, rx->page_chunks);
+	rx->rx_bufs_count = prio_bufs + small_pkts_buf_count;
+	rx->rx_bufs = vzalloc(rx->rx_bufs_count * sizeof(*rx->rx_bufs));
+	if (rx->rx_bufs == NULL) {
+		rc = -ENOMEM;
+		goto err;
+	}
+
+	for (i = 0; i < rx->rx_bufs_count; i++) {
+		struct rx_buffer *buf = &rx->rx_bufs[i];
+
+		buf->id = i;
+		list_add_tail(&buf->buf_list, &rx->rx_unallocated);
+	}
+
+	/* Allocate RX buffers */
+	rc = alloc_rx_buffers(rx, prio_bufs, C_PTL_LIST_PRIORITY);
+	if (rc == 0) {
+		netdev_info(dev->ndev, "Cannot allocate RX priority buffers\n");
+		rc = -ENOMEM;
+		goto err_free_rx_bufs;
+	} else if (rc != prio_bufs) {
+		netdev_info(dev->ndev,
+			    "Only allocated %d RX priority buffers out of %u requested\n",
+			    rc, prio_bufs);
+	}
+
+	rc = alloc_rx_buffers(rx, small_pkts_buf_count, C_PTL_LIST_REQUEST);
+	if (rc == 0) {
+		netdev_info(dev->ndev, "Cannot allocate RX request buffers\n");
+		rc = -ENOMEM;
+		goto err_free_buffers;
+	} else if (rc != small_pkts_buf_count) {
+		netdev_info(dev->ndev,
+			    "Only allocated %d RX request buffers out of %d requested\n",
+			    rc, small_pkts_buf_count);
+	}
+
+	cq_alloc_opts.policy = CXI_CQ_UPDATE_HIGH_FREQ_EMPTY;
+	cq_alloc_opts.count = dev->ringparam.rx_pending + TGT_CQ_OVERHEAD_COUNT;
+	cq_alloc_opts.lpe_cdt_thresh_id = lpe_cdt_thresh_id;
+	rx->cq_tgt_prio = cxi_cq_alloc(dev->lni, dev->err_eq, &cq_alloc_opts,
+				       rx->node);
+	if (IS_ERR(rx->cq_tgt_prio)) {
+		rc = PTR_ERR(rx->cq_tgt_prio);
+		goto err_free_buffers;
+	}
+
+	rx->eq_buf_size = get_rxq_eq_buf_size(dev);
+	rx->eq_buf = kvzalloc_node(rx->eq_buf_size, GFP_KERNEL, rx->node);
+	if (!rx->eq_buf) {
+		rc = -ENOMEM;
+		goto err_free_cq;
+	}
+
+	if (is_vmalloc_addr(rx->eq_buf)) {
+		rx->eq_md = cxi_map(dev->lni, (uintptr_t)rx->eq_buf,
+				    rx->eq_buf_size, CXI_MAP_WRITE, NULL);
+		if (IS_ERR(rx->eq_md)) {
+			rc = PTR_ERR(rx->eq_md);
+			goto err_free_eq_buf;
+		}
+	} else {
+		eq_attr.flags = CXI_EQ_PASSTHROUGH;
+	}
+
+	eq_attr.queue = rx->eq_buf;
+	eq_attr.queue_len = rx->eq_buf_size;
+	eq_attr.cpu_affinity = rx->cpu;
+	rx->eq = cxi_eq_alloc(dev->lni, rx->eq_md, &eq_attr, rx_eq_cb, rx, NULL,
+			      NULL);
+	if (IS_ERR(rx->eq)) {
+		rc = PTR_ERR(rx->eq);
+		goto err_unmap_eq_buf;
+	}
+
+	cxi_eq_int_disable(rx->eq);
+
+	/* Create an Ethernet portal */
+	rx->pt = cxi_pte_alloc(dev->lni, rx->eq, &pt_alloc_opts);
+	if (IS_ERR(rx->pt)) {
+		rc = PTR_ERR(rx->pt);
+		goto err_free_eq;
+	}
+
+	NETIF_NAPI_ADD(dev->ndev, &rx->napi, cxi_rx_eth_poll);
+
+	return 0;
+
+err_free_eq:
+	cxi_eq_free(rx->eq);
+err_unmap_eq_buf:
+	if (is_vmalloc_addr(rx->eq_buf))
+		cxi_unmap(rx->eq_md);
+err_free_eq_buf:
+	kvfree(rx->eq_buf);
+err_free_cq:
+	cxi_cq_free(rx->cq_tgt_prio);
+err_free_buffers:
+	free_rx_buffers(rx);
+err_free_rx_bufs:
+	vfree(rx->rx_bufs);
+err:
+	rx->dev = NULL;
+	return rc;
+}
+
+void free_rx_queue(struct rx_queue *rx)
+{
+	if (!rx->dev)
+		return;
+
+	netif_napi_del(&rx->napi);
+
+	cxi_pte_free(rx->pt);
+	rx->pt = NULL;
+	cxi_eq_free(rx->eq);
+	rx->eq = NULL;
+
+	if (is_vmalloc_addr(rx->eq_buf))
+		cxi_unmap(rx->eq_md);
+	kvfree(rx->eq_buf);
+	rx->eq_md = NULL;
+	rx->eq_buf = NULL;
+
+	cxi_cq_free(rx->cq_tgt_prio);
+	rx->cq_tgt_prio = NULL;
+	free_rx_buffers(rx);
+	vfree(rx->rx_bufs);
+	rx->rx_bufs = NULL;
+	rx->dev = NULL;
+}
+
+void enable_rx_queue(struct rx_queue *rx)
+{
+	napi_enable(&rx->napi);
+	cxi_eq_int_enable(rx->eq);
+}
+
+void disable_rx_queue(struct rx_queue *rx)
+{
+	if (!rx->dev)
+		return;
+
+	cxi_eq_int_disable(rx->eq);
+	napi_disable(&rx->napi);
+}
+
+#define DEFAULT_RESERVED_LES 1024U
+
+static unsigned int get_reserved_les(void)
+{
+	int reserved_les;
+	int max_rx_queues = max_rss_queues + 1; // +1 for PTP queue
+	int max_les_per_queue;
+
+	reserved_les = cxi_get_lpe_append_credits(lpe_cdt_thresh_id);
+	if (reserved_les < 0) {
+		pr_err("Failed to get LPE append credits. Used default %u\n",
+		       DEFAULT_RESERVED_LES);
+		return DEFAULT_RESERVED_LES;
+	}
+
+	max_les_per_queue = reserved_les + small_pkts_buf_count;
+
+	/* Each RX queue needs max_les_per_queue. Since the CXI service reserves
+	 * LEs against each LPE pool (4 pools total), the total number of LEs
+	 * needed needs to be divided by 4. cxi_pte_alloc() will then round
+	 * robind across the LPE pools.
+	 */
+	reserved_les = max_les_per_queue * (max_rx_queues / C_PE_COUNT);
+
+	/* Need to account if the number of queues is not a factor of 4. */
+	if (max_rx_queues % C_PE_COUNT)
+		reserved_les += max_les_per_queue;
+
+	return reserved_les;
+}
+
+/* Allocate the hardware resources and some receive buffers */
+int hw_setup(struct cxi_eth *dev)
+{
+	int rc;
+	int lac;
+	struct net_device *ndev = dev->ndev;
+	const struct cxi_rsrc_limits limits = {
+		.acs = {
+			.max = 1,
+			.res = 1,
+		},
+		.eqs = {
+			.max = 1 + max_rss_queues + (max_tx_queues * 2),
+			.res = 1,
+		},
+		.cts = {
+			.max = 0,
+			.res = 0,
+		},
+		.ptes = {
+			.max = 1 + max_rss_queues,
+			.res = 8,
+		},
+		.txqs = {
+			.max = max_tx_queues * 2,
+			.res = 1,
+		},
+		.tgqs = {
+			.max = 1 + 1 + max_rss_queues,
+			.res = 1,
+		},
+		.tles = {
+			.max = 0,
+			.res = 0,
+		},
+		/* Will actually have 4 * res value for les */
+		.les = {
+			.max = MAX_LE_LIMIT,
+			.res = get_reserved_les(),
+		},
+	};
+	struct cxi_svc_desc svc_desc = {
+		.resource_limits = true,
+		.limits = limits,
+		.is_system_svc = true,
+		.restricted_members = true,
+		.restricted_vnis = true,
+		.num_vld_vnis = 0,
+		.enable = true,
+		.members[0] = {
+			.type = CXI_SVC_MEMBER_UID,
+			.svc_member.uid = (current_euid()).val,
+		},
+	};
+	struct cxi_cq_alloc_opts cq_alloc_opts = {};
+	u8 shared_cp_pcp;
+
+	/* Allocate a Service */
+	rc = cxi_svc_alloc(dev->cxi_dev, &svc_desc, NULL, "ethernet-svc");
+	if (rc < 0) {
+		netdev_info(ndev, "Can't reserve resources: %d\n", rc);
+		goto err;
+	}
+	dev->svc_id = rc;
+
+	dev->lni = cxi_lni_alloc(dev->cxi_dev, dev->svc_id);
+	if (IS_ERR(dev->lni)) {
+		rc = PTR_ERR(dev->lni);
+		netdev_info(ndev, "Can't get an LNI: %d\n", rc);
+		goto err_free_svc;
+	}
+
+	lac = cxi_phys_lac_alloc(dev->lni);
+	if (lac < 0) {
+		rc = -EINVAL;
+		netdev_info(ndev, "failed to get a physical LAC: %d\n", rc);
+		goto err_free_ni;
+	}
+	dev->phys_lac = lac;
+
+	/* Store PCPs for all the Eth Classes */
+	dev->eth1_pcp = cxi_get_tc_req_pcp(dev->cxi_dev, CXI_ETH_TC1);
+	if (dev->eth1_pcp < 0) {
+		rc = -EINVAL;
+		goto err_free_ni;
+	}
+
+	/* Determine if Eth2 is active. Eth1 and Shared will always be active */
+	dev->eth2_pcp = cxi_get_tc_req_pcp(dev->cxi_dev, CXI_ETH_TC2);
+	dev->eth2_active = (dev->eth2_pcp >= 0 && dev->eth2_pcp < 8);
+
+	/* The Shared Ethernet Class is configured with all PCPs apart from
+	 * the ones used by Eth1 and Eth2. Use any of them for configuring
+	 * the CP
+	 */
+	for (shared_cp_pcp = 0; shared_cp_pcp <= 7; shared_cp_pcp++) {
+		if (shared_cp_pcp != dev->eth1_pcp &&
+		    (!dev->eth2_active || shared_cp_pcp != dev->eth2_pcp))
+			break;
+	}
+
+	/* Allocate CPs for each Eth TC
+	 * TODO: Support unique communication profiles + CQ + MCUs for each
+	 * PCP. This may not be feasible with the resource needed for each.
+	 */
+	dev->eth1_cp = cxi_trig_cp_alloc(dev->lni, dev->eth1_pcp,
+					 CXI_ETH_TC1, CXI_TC_TYPE_DEFAULT,
+					 NON_TRIG_LCID);
+	if (IS_ERR(dev->eth1_cp)) {
+		rc = PTR_ERR(dev->eth1_cp);
+		netdev_info(ndev, "Can't allocate Eth1 CP: %d\n", rc);
+		goto err_free_lac;
+	}
+
+	dev->shared_cp = cxi_trig_cp_alloc(dev->lni, shared_cp_pcp,
+					   CXI_ETH_SHARED,
+					   CXI_TC_TYPE_DEFAULT, NON_TRIG_LCID);
+	if (IS_ERR(dev->shared_cp)) {
+		rc = PTR_ERR(dev->shared_cp);
+		netdev_info(ndev, "Can't allocate Eth shared CP: %d\n", rc);
+		goto err_free_eth1_cp;
+	}
+
+	if (dev->eth2_active) {
+		dev->eth2_cp = cxi_trig_cp_alloc(dev->lni, dev->eth2_pcp,
+						 CXI_ETH_TC2,
+						 CXI_TC_TYPE_DEFAULT,
+						 NON_TRIG_LCID);
+		if (IS_ERR(dev->eth2_cp)) {
+			rc = PTR_ERR(dev->eth2_cp);
+			netdev_info(ndev, "Can't allocate Eth2 CP: %d\n", rc);
+			goto err_free_shared_cp;
+		}
+	}
+
+	dev->err_eq_attr.queue_len = PAGE_SIZE;
+	dev->err_eq_attr.queue = (void *)
+		__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+				 get_order(dev->err_eq_attr.queue_len));
+	if (!dev->err_eq_attr.queue) {
+		rc = -ENOMEM;
+		netdev_info(ndev, "Can't allocate the EQ buffer: %d\n", rc);
+		goto err_free_eth2_cp;
+	}
+
+	dev->err_eq_attr.flags = CXI_EQ_PASSTHROUGH;
+	dev->err_eq = cxi_eq_alloc(dev->lni, NULL, &dev->err_eq_attr, err_eq_cb,
+				   dev, NULL, NULL);
+	if (IS_ERR(dev->err_eq)) {
+		rc = PTR_ERR(dev->err_eq);
+		netdev_info(ndev, "Can't allocate the EQ: %d\n", rc);
+		goto err_free_err_eq_buf;
+	}
+
+	cq_alloc_opts.policy = CXI_CQ_UPDATE_HIGH_FREQ_EMPTY;
+	cq_alloc_opts.count =
+		(small_pkts_buf_count * (max_rss_queues + 1)) +	TGT_CQ_OVERHEAD_COUNT;
+	dev->cq_tgt_req = cxi_cq_alloc(dev->lni, dev->err_eq, &cq_alloc_opts,
+				       dev_to_node(&dev->cxi_dev->pdev->dev));
+	if (IS_ERR(dev->cq_tgt_req)) {
+		rc = PTR_ERR(dev->cq_tgt_req);
+		netdev_info(ndev, "Can't allocate the target CQ: %d\n", rc);
+		goto err_free_err_eq;
+	}
+
+	/* Configure RXQ zero used for default/catch-all traffic. */
+	rc = alloc_rx_queue(dev, 0);
+	if (rc) {
+		netdev_info(ndev, "Can't allocate the receive queue: %d\n", rc);
+		goto err_free_cq_tgt_req;
+	}
+	dev->res.ptn_def = dev->rxqs[0].pt->id;
+
+	rc = post_rx_buffers(&dev->rxqs[0], GFP_KERNEL);
+	if (rc < 0) {
+		netdev_info(ndev, "Cannot post any RX buffers: %d\n", rc);
+		goto err_free_rx_queue;
+	}
+
+	/* Configure PTP RX queue */
+	rc = alloc_rx_queue(dev, PTP_RX_Q);
+	if (rc) {
+		netdev_info(ndev, "Can't allocate the PTP receive queue: %d\n",
+			    rc);
+		goto err_free_rx_queue;
+	}
+	dev->res.ptn_ptp = dev->rxqs[PTP_RX_Q].pt->id;
+	rc = post_rx_buffers(&dev->rxqs[PTP_RX_Q], GFP_KERNEL);
+	if (rc < 0) {
+		netdev_info(ndev, "Cannot post RX buffers: %d\n", rc);
+		goto err_free_ptp_queue;
+	}
+
+	rc = alloc_tx_queue(dev, 0);
+	if (rc) {
+		netdev_info(ndev, "Can't allocate the transmit queue: %d\n",
+			    rc);
+		goto err_free_ptp_queue;
+	}
+
+	dev->mac_addr = ether_addr_to_u64(ndev->dev_addr);
+	dev->ptp_mac_addr = PTP_L2_MAC;
+
+	cxi_eth_add_mac(dev->cxi_dev, &dev->res, dev->mac_addr,	false);
+
+	enable_rx_queue(&dev->rxqs[0]);
+	enable_rx_queue(&dev->rxqs[PTP_RX_Q]);
+	enable_tx_queue(&dev->txqs[0]);
+
+	dev->res.rss_queues = 1;
+	rc = cxi_set_rx_channels(dev, dev->ndev->real_num_rx_queues);
+	if (rc) {
+		netdev_info(ndev,
+			    "Can't set the number of RX channels to %d: %d\n",
+			    dev->ndev->real_num_rx_queues, rc);
+		goto err_disable_queues;
+	}
+
+	dev->cur_txqs = 1;
+	rc = cxi_set_tx_channels(dev, dev->ndev->real_num_tx_queues);
+	if (rc) {
+		netdev_info(ndev,
+			    "Can't set the number of TX channels to %d: %d\n",
+			    dev->ndev->real_num_tx_queues, rc);
+		goto err_set_rx_queues;
+	}
+
+	cxi_eth_set_rx_mode(ndev);
+	netif_tx_start_all_queues(ndev);
+	dev->is_active = true;
+
+	return 0;
+
+err_set_rx_queues:
+	cxi_set_rx_channels(dev, 1);
+err_disable_queues:
+	disable_tx_queue(&dev->txqs[0]);
+	disable_rx_queue(&dev->rxqs[PTP_RX_Q]);
+	disable_rx_queue(&dev->rxqs[0]);
+	cxi_eth_set_list_invalidate_all(dev->cxi_dev, &dev->res);
+	free_tx_queue(&dev->txqs[0]);
+err_free_ptp_queue:
+	free_rx_queue(&dev->rxqs[PTP_RX_Q]);
+err_free_rx_queue:
+	free_rx_queue(&dev->rxqs[0]);
+err_free_cq_tgt_req:
+	cxi_cq_free(dev->cq_tgt_req);
+err_free_err_eq:
+	cxi_eq_free(dev->err_eq);
+err_free_err_eq_buf:
+	free_pages((unsigned long)dev->err_eq_attr.queue,
+		   get_order(dev->err_eq_attr.queue_len));
+err_free_eth2_cp:
+	if (dev->eth2_active)
+		cxi_cp_free(dev->eth2_cp);
+err_free_shared_cp:
+	cxi_cp_free(dev->shared_cp);
+err_free_eth1_cp:
+	cxi_cp_free(dev->eth1_cp);
+err_free_lac:
+	cxi_phys_lac_free(dev->lni, dev->phys_lac);
+err_free_ni:
+	cxi_lni_free(dev->lni);
+err_free_svc:
+	cxi_svc_destroy(dev->cxi_dev, dev->svc_id);
+err:
+	return rc;
+}
+
+/* Release hardware resources belonging to a device. */
+void hw_cleanup(struct cxi_eth *dev)
+{
+	int i;
+
+	if (!dev->is_active)
+		return;
+
+	netif_tx_stop_all_queues(dev->ndev);
+
+	dev->is_active = false;
+
+	for (i = 0; i < dev->cur_txqs; i++)
+		disable_tx_queue(&dev->txqs[i]);
+
+	/* Disable RSS support and free RX queues. */
+	cxi_eth_clear_indir_table(dev->cxi_dev, &dev->res);
+	for (i = 0; i < dev->res.rss_queues; i++)
+		disable_rx_queue(&dev->rxqs[i]);
+	disable_rx_queue(&dev->rxqs[PTP_RX_Q]);
+
+	cxi_eth_set_list_invalidate_all(dev->cxi_dev, &dev->res);
+
+	for (i = 0; i < dev->cur_txqs; i++)
+		free_tx_queue(&dev->txqs[i]);
+
+	for (i = 0; i < dev->res.rss_queues; i++)
+		free_rx_queue(&dev->rxqs[i]);
+	free_rx_queue(&dev->rxqs[PTP_RX_Q]);
+
+	cxi_cq_free(dev->cq_tgt_req);
+	dev->cq_tgt_req = NULL;
+	cxi_eq_free(dev->err_eq);
+	free_pages((unsigned long)dev->err_eq_attr.queue,
+		   get_order(dev->err_eq_attr.queue_len));
+	dev->err_eq = NULL;
+	if (dev->eth2_active) {
+		cxi_cp_free(dev->eth2_cp);
+		dev->eth2_cp = NULL;
+	}
+	cxi_cp_free(dev->shared_cp);
+	dev->shared_cp = NULL;
+	cxi_cp_free(dev->eth1_cp);
+	dev->eth1_cp = NULL;
+	cxi_phys_lac_free(dev->lni, dev->phys_lac);
+	dev->phys_lac = 0;
+	cxi_lni_free(dev->lni);
+	dev->lni = NULL;
+	cxi_svc_destroy(dev->cxi_dev, dev->svc_id);
+}
+
+static struct sk_buff *eth_rx_copy(struct rx_queue *rx,
+				   struct rx_buffer *buf,
+				   const struct c_event_target_enet *event)
+{
+	struct cxi_eth *dev = rx->dev;
+	struct net_device *ndev = dev->ndev;
+	struct sk_buff *skb;
+	struct page *page;
+	unsigned char *start;
+	u64 buffer_start;
+	u64 offset;
+
+	if (unlikely(!buf->md || !buf->data))
+		return NULL;
+	buffer_start = CXI_VA_TO_IOVA(buf->md, (uintptr_t)buf->data);
+	if (unlikely(event->start < buffer_start))
+		return NULL;
+	offset = event->start - buffer_start;
+	if (unlikely(offset > buf->data_size ||
+		     event->length > buf->data_size - offset))
+		return NULL;
+	start = buf->data + offset;
+
+	if (likely(rx->frag_state == FRAG_NONE && !event->more_frags)) {
+		/* single fragment */
+		skb = napi_alloc_skb(&rx->napi, event->length);
+		if (unlikely(!skb))
+			return NULL;
+
+		__skb_put_data(skb, start, event->length);
+
+		skb->protocol = eth_type_trans(skb, ndev);
+	} else {
+		skb = napi_get_frags(&rx->napi);
+		if (unlikely(!skb))
+			return NULL;
+
+		/* The SKB may already be full, due to the adapter
+		 * being configured to fragment too much.
+		 */
+		if (unlikely(skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS))
+			return NULL;
+
+		if (rx->frag_state == FRAG_NONE &&
+		    event->length <= skb_availroom(skb)) {
+			__skb_put_data(skb, start, event->length);
+		} else {
+			if (unlikely(event->length > PAGE_SIZE))
+				return NULL;
+			page = alloc_page(GFP_ATOMIC);
+			if (unlikely(!page)) {
+				napi_free_frags(&rx->napi);
+				return NULL;
+			}
+
+			memcpy(page_address(page), start, event->length);
+
+			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+					page, 0, event->length, PAGE_SIZE);
+		}
+	}
+
+	return skb;
+}
+
+static struct sk_buff *eth_rx_frags(struct rx_queue *rx,
+				    struct rx_buffer *buf,
+				    const struct c_event_target_enet *event)
+{
+	struct cxi_eth *eth_dev = rx->dev;
+	struct device *dev = &eth_dev->cxi_dev->pdev->dev;
+	struct sk_buff *skb;
+	struct page *page;
+	int frag_len;
+
+	if (unlikely(!buf->page || !buf->posted_count ||
+		     buf->cur_offset >= PAGE_SIZE ||
+		     event->length > PAGE_SIZE - buf->cur_offset ||
+		     event->length > eth_dev->eth_info.max_segment_size))
+		return NULL;
+
+	skb = napi_get_frags(&rx->napi);
+	if (unlikely(!skb))
+		return NULL;
+
+	/* The SKB may already be full, due to the adapter
+	 * being configured to fragment too much.
+	 */
+	if (unlikely(skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS))
+		return NULL;
+
+	page = buf->page;
+	frag_len = event->length;
+
+	if (rx->frag_state == FRAG_NONE)
+		prefetch(page_address(page));
+
+	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+			page, buf->cur_offset, frag_len,
+			eth_dev->eth_info.max_segment_size);
+	dma_sync_single_for_cpu(dev, buf->mapping + buf->cur_offset, frag_len,
+				DMA_FROM_DEVICE);
+
+	buf->cur_offset += eth_dev->eth_info.max_segment_size;
+	buf->posted_count--;
+
+	if (buf->posted_count == 0) {
+		dma_unmap_page(dev, dma_unmap_addr(buf, mapping), PAGE_SIZE,
+			       DMA_FROM_DEVICE);
+		put_page(page);
+		buf->page = NULL;
+	}
+
+	return skb;
+}
+
+/* Check whether the RoCE packet in the SKB has been segmented by
+ * hardware, which also removed its iCRC. Fix the headers if that
+ * happened.
+ */
+static void check_segmented_roce(struct sk_buff *skb)
+{
+	int frag_len;
+	u16 protocol;
+	struct ethhdr *ethhdr;
+	size_t vhdr_size = 0;
+
+	/* heuristic to figure out whether Cassini stripped the iCRC
+	 * from a RoCE packet, following its segmentation.
+	 */
+	if (skb_shinfo(skb)->nr_frags != 1)
+		return;
+
+	ethhdr = (void *)skb->data;
+	protocol = be16_to_cpu(ethhdr->h_proto);
+
+	/* Check for vlan header - we will support at most one vlan header,
+	 * no nested headers, ipv6 extensions, etc.
+	 */
+	if (eth_type_vlan(ethhdr->h_proto)) {
+		struct vlan_ethhdr *veth;
+
+		veth = (struct vlan_ethhdr *)skb->data;
+		protocol = be16_to_cpu(veth->h_vlan_encapsulated_proto);
+		vhdr_size = sizeof(struct vlan_hdr);
+	}
+
+	/* Check length. ETH+IPv4/6+UDP+BTH headers. */
+	if (skb_headlen(skb) < (54 + vhdr_size) || skb_headlen(skb) > (90 + vhdr_size))
+		return;
+
+	frag_len = skb_frag_size(&skb_shinfo(skb)->frags[0]);
+	if (frag_len != 1024 && frag_len != 2048 && frag_len != 4096)
+		return;
+
+	/* Adjust the IP packet by removing the 4 bytes and
+	 * recomputing the IP header checksum.
+	 */
+	switch (protocol) {
+	case ETH_P_IP: {
+		struct udphdr *udph;
+		struct iphdr *iph;
+
+		iph = (void *)skb->data + sizeof(struct ethhdr) + vhdr_size;
+		iph->tot_len = cpu_to_be16(be16_to_cpu(iph->tot_len) - 4);
+		iph->check = 0;
+		iph->check = ip_fast_csum((const void *)iph, iph->ihl);
+
+		udph = (void *)(iph + 1);
+		udph->len = cpu_to_be16(be16_to_cpu(udph->len) - 4);
+	}
+		break;
+
+	case ETH_P_IPV6: {
+		struct udphdr *udph;
+		struct ipv6hdr *ip6h;
+
+		ip6h = (void *)skb->data + sizeof(struct ethhdr) + vhdr_size;
+		ip6h->payload_len =
+			cpu_to_be16(be16_to_cpu(ip6h->payload_len) - 4);
+
+		if (ip6h->nexthdr != NEXTHDR_UDP) {
+			netdev_warn_once(skb->dev,
+					 "Unexpected ipv6 packet format (nexthdr = %d)\n",
+					 ip6h->nexthdr);
+			return;
+		}
+
+		udph = (void *)(ip6h + 1);
+		udph->len = cpu_to_be16(be16_to_cpu(udph->len) - 4);
+	}
+		break;
+
+	default:
+		netdev_warn_once(skb->dev,
+				 "unexpected protocol in RoCE packet: %x\n",
+				 protocol);
+		return;
+	}
+
+	/* TODO. For now, use a bit in the SKB to tell RXE that
+	 * the iCRC is gone, and that the packet is otherwise correct.
+	 */
+	cxi_skb_icrc_gone(skb);
+}
+
+/* Convert from hash type (i.e. C_RSS_HASH_IPV4_UDP) to linux hash
+ * type (i.e. PKT_HASH_TYPE_L4). Only valid if the hash type is not
+ * C_RSS_HASH_NONE.
+ */
+static const u8 rss_hash_type_to_level[C_RSS_HASH_IPV6_FLOW_LABEL + 1] = {
+	[C_RSS_HASH_IPV4] = PKT_HASH_TYPE_L3,
+	[C_RSS_HASH_IPV4_PROTOCOL] = PKT_HASH_TYPE_L3,
+	[C_RSS_HASH_IPV4_FLOW_LABEL] = PKT_HASH_TYPE_L3,
+	[C_RSS_HASH_IPV6] = PKT_HASH_TYPE_L3,
+	[C_RSS_HASH_IPV6_PROTOCOL] = PKT_HASH_TYPE_L3,
+	[C_RSS_HASH_IPV6_FLOW_LABEL] = PKT_HASH_TYPE_L3,
+
+	[C_RSS_HASH_IPV4_TCP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV4_UDP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV4_PROTOCOL_TCP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV4_PROTOCOL_UDP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV4_PROTOCOL_UDP_ROCE] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV6_TCP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV6_UDP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV6_PROTOCOL_TCP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV6_PROTOCOL_UDP] = PKT_HASH_TYPE_L4,
+	[C_RSS_HASH_IPV6_PROTOCOL_UDP_ROCE] = PKT_HASH_TYPE_L4,
+};
+
+static void set_rss_hash_value(const struct c_event_target_enet *event,
+			       struct sk_buff *skb)
+{
+	if (event->rss_hash_type == C_RSS_HASH_NONE ||
+	    event->rss_hash_type > C_RSS_HASH_IPV6_FLOW_LABEL)
+		return;
+
+	skb_set_hash(skb, event->rss_hash_value,
+		     rss_hash_type_to_level[event->rss_hash_type]);
+}
+
+/* Process a received packet */
+static bool eth_receive(struct rx_queue *rx,
+			const struct c_event_target_enet *event)
+{
+	struct cxi_eth *dev = rx->dev;
+	struct net_device *ndev = dev->ndev;
+	struct sk_buff *skb = NULL;
+	struct rx_buffer *buf;
+	bool packet_complete = false;
+
+	/* Find the target buffer */
+	if (event->ptlte_index != rx->pt->id) {
+		netdev_err(ndev, "Unknown portal target in RX event (%d instead of %d)\n",
+			   event->ptlte_index, rx->pt->id);
+		return packet_complete;
+	}
+
+	buf = &rx->rx_bufs[event->buffer_id];
+
+	if (unlikely(event->return_code != C_RC_OK ||
+		     (event->is_roce && !event->more_frags &&
+		      !event->roce_icrc_ok) ||
+		     event->parser_err ||
+		     rx->frag_state == FRAG_DROP)) {
+		goto drop;
+	}
+
+	if (event->ptl_list == C_PTL_LIST_PRIORITY)
+		skb = eth_rx_frags(rx, buf, event);
+	else
+		skb = eth_rx_copy(rx, buf, event);
+
+	if (unlikely(!skb))
+		goto drop;
+
+	if (dev->is_c2) {
+		if (event->seg_cnt == 0) {
+			/* It's a new packet */
+			rx->pkt_cnt = event->pkt_cnt;
+			rx->next_seg_cnt = 1;
+
+			/* Free existing incomplete segments if any */
+			if (rx->frag_state != FRAG_NONE) {
+				ndev->stats.rx_dropped++;
+				rx->frag_state = FRAG_NONE;
+				napi_free_frags(&rx->napi);
+			}
+		} else if (event->pkt_cnt != rx->pkt_cnt ||
+			   event->seg_cnt != rx->next_seg_cnt) {
+			goto drop;
+		} else {
+			rx->next_seg_cnt++;
+		}
+	}
+
+	if (event->more_frags == 0) {
+		__be16 csum;
+
+		ndev->stats.rx_packets++;
+		ndev->stats.rx_bytes += skb->len;
+
+		csum = (__force __be16)event->checksum;
+		skb->csum = (__force __wsum)be16_to_cpu(csum);
+		skb->ip_summed = CHECKSUM_COMPLETE;
+
+		/* Cassini ERRATA-3258. This should test event->timestamp, but
+		 * Cassini 1/1.1 doesn't set it.
+		 */
+		if (dev->ptp_ts_enabled &&
+		    event->ptlte_index == dev->res.ptn_ptp) {
+			struct c_ts ts;
+			int rc;
+
+			rc = skb_copy_bits(skb, skb->len - sizeof(ts),
+					   &ts, sizeof(ts));
+			if (rc)
+				goto drop;
+
+			/* Cassini ERRATA-3276. Ignore the timestamp on Cassini 1
+			 * if ns is close to 1s, as RX timestamp can
+			 * be off by one second in that case. No issue
+			 * on Cassini 2.
+			 */
+			if (dev->is_c2 || ts.ns < 999999700)
+				skb_hwtstamps(skb)->hwtstamp =
+					ktime_set(ts.sec, ts.ns);
+
+			pskb_trim(skb, skb->len - sizeof(ts));
+		}
+
+		if (event->is_roce) {
+			if (skb->len <= 64 && !dev->is_c2 &&
+			    (skb_shinfo(skb)->nr_frags == 0))
+				/* Due to a rarely occurring hardware
+				 * bug, iCRC on packets smaller than
+				 * 65 bytes may be passed although
+				 * they are corrupted.  Tell the rxe
+				 * driver to check it again.
+				 */
+				cxi_force_icrc_check(skb);
+			else if (dev->priv_flags & CXI_ETH_PF_ROCE_OPT)
+				check_segmented_roce(skb);
+		}
+
+		skb_record_rx_queue(skb, rx->id);
+
+		if (ndev->features & NETIF_F_RXHASH)
+			set_rss_hash_value(event, skb);
+
+		if (skb_is_nonlinear(skb)) {
+			napi_gro_frags(&rx->napi);
+
+			rx->frag_state = FRAG_NONE;
+		} else {
+			napi_gro_receive(&rx->napi, skb);
+		}
+
+		packet_complete = true;
+	} else {
+		rx->frag_state = FRAG_MORE;
+	}
+
+requeue:
+	if (event->ptl_list == C_PTL_LIST_PRIORITY) {
+		rx->stats.unlinked_prio++;
+
+		if (buf->page == NULL) {
+			list_del(&buf->buf_list);
+			list_add_tail(&buf->buf_list, &rx->rx_ready);
+		}
+	} else if (event->auto_unlinked) {
+		rx->stats.unlinked_req++;
+		list_del(&buf->buf_list);
+		list_add_tail(&buf->buf_list, &rx->rx_ready);
+	}
+
+	return packet_complete;
+
+drop:
+	/* Error on receive, NIC dropped this frag, or driver dropped a previous
+	 * frag, or skb allocation failure
+	 */
+	netdev_dbg(dev->ndev,
+		   "received return_code=%d more=%d state=%d ptl_list=%d roce=%d\n",
+		   event->return_code, event->more_frags, rx->frag_state,
+		   event->ptl_list, event->is_roce);
+
+	if (event->return_code != C_RC_OK)
+		rx->last_bad_frag_drop_rc = event->return_code;
+
+	/* Not always a whole packet, so in theory a single packet
+	 * could be counted several times, one for each segment.
+	 */
+	ndev->stats.rx_dropped++;
+
+	if (rx->frag_state != FRAG_DROP)
+		napi_free_frags(&rx->napi);
+
+	if (event->more_frags)
+		rx->frag_state = FRAG_DROP;
+	else
+		rx->frag_state = FRAG_NONE;
+
+	goto requeue;
+}
+
+/* RX queue interrupt handler. Disable the interrupts for that EQ and
+ * trigger NAPI.
+ */
+static void rx_eq_cb(void *context)
+{
+	struct rx_queue *rx = context;
+
+	rx->time_napi_schedule = ktime_get_raw();
+	napi_schedule(&rx->napi);
+}
+
+static void add_ktime_to_bucket(struct bucket *bucket, u64 ns)
+{
+	int i;
+
+	if (ns > bucket->max_ts)
+		bucket->max_ts = ns;
+	if (ns < bucket->min_ts)
+		bucket->min_ts = ns;
+
+	i = fls(ns + 1);
+	if (i >= NB_BUCKETS)
+		i = NB_BUCKETS - 1;
+
+	bucket->b[i]++;
+}
+
+#define RX_ETH_EQ_UNACKED_MAX 16U
+
+/* NAPI poll function.
+ * Only the receive event will be counted towards the budget.
+ */
+static int cxi_rx_eth_poll(struct napi_struct *napi, int budget)
+{
+	struct rx_queue *rx = container_of(napi, struct rx_queue, napi);
+	struct cxi_eth *dev = rx->dev;
+	const union c_event *event;
+	int work_done = 0;
+	int event_count = 0;
+	bool reschedule = false;
+	bool packet_processed = false;
+	int rc;
+
+	if (rx->time_napi_schedule) {
+		ktime_t enter_ts;
+		u64 time_exec;
+
+		enter_ts = ktime_get_raw();
+		time_exec = ktime_to_ns(ktime_sub(enter_ts,
+						  rx->time_napi_schedule));
+		add_ktime_to_bucket(&rx->eth_napi_schedule, time_exec);
+
+		/* Prevent adding time when the function is re-entered */
+		rx->time_napi_schedule = 0;
+	}
+
+	while (work_done < budget &&
+	       (event = cxi_eq_get_event(rx->eq))) {
+		switch (event->hdr.event_type) {
+		case C_EVENT_LINK:
+			packet_processed = false;
+			netdev_warn(dev->ndev, "LE append failure: %d\n",
+				    event->tgt_long.return_code);
+			break;
+		case C_EVENT_ETHERNET:
+			packet_processed = eth_receive(rx, &event->enet);
+			if (packet_processed)
+				work_done++;
+			break;
+		case C_EVENT_STATE_CHANGE:
+			packet_processed = false;
+			netdev_warn(dev->ndev, "State change (%d, %d, %d)\n",
+				    event->tgt_long.ptlte_index,
+			       event->tgt_long.initiator.state_change.ptlte_state,
+			       event->tgt_long.return_code);
+			break;
+		default:
+			packet_processed = false;
+			netdev_warn(dev->ndev, "Unhandled event %d, rc %d\n",
+				    event->hdr.event_type,
+				    event->hdr.return_code);
+			print_hex_dump(KERN_INFO,
+				       "cxi eth unexpected target event: ",
+				       DUMP_PREFIX_ADDRESS, 16, 1, event,
+				       (8 << event->hdr.event_size), false);
+		}
+
+		event_count++;
+		if ((event_count >= RX_ETH_EQ_UNACKED_MAX) &&
+		    packet_processed) {
+			cxi_eq_int_disable(rx->eq);
+			rc = post_rx_buffers(rx, GFP_ATOMIC);
+			if (rc < 0)
+				reschedule = true;
+
+			event_count = 0;
+		}
+	}
+
+	BUG_ON(work_done > budget);
+
+	/* ACK all EQ space before appending new buffers. */
+	if (event_count)
+		cxi_eq_int_disable(rx->eq);
+
+	if (!list_empty(&rx->rx_ready)) {
+		rc = post_rx_buffers(rx, GFP_ATOMIC);
+		if (rc < 0)
+			reschedule = true;
+	}
+
+	if (work_done < budget && napi_complete_done(napi, work_done)) {
+		/* Only re-enable interrupts if NAPI is not going to be
+		 * rescheduled.
+		 */
+		if (reschedule) {
+			netdev_warn_once(dev->ndev,
+					 "Forcing NAPI reschedule\n");
+			napi_schedule(&rx->napi);
+		} else {
+			cxi_eq_int_enable(rx->eq);
+		}
+	}
+
+	return work_done;
+}
+
+/* Hardware defined RSS hash bit is one less than the corresponding
+ * c_rss_hash_type enum value. This macro should only be used if the
+ * c_rss_hash_type enum value is not C_RSS_HASH_NONE.
+ */
+#define RSS_HASH_BIT(hash_type) ((hash_type) - 1)
+
+int cxi_eth_open(struct net_device *ndev)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct hwtstamp_config ts_cfg = {
+		.flags = 0,
+		.tx_type = HWTSTAMP_TX_OFF,
+		.rx_filter = HWTSTAMP_FILTER_NONE,
+	};
+
+	cxi_set_eth_name(dev->cxi_dev, netdev_name(ndev));
+
+	/* The indirection base is always 0 for now, as there is only
+	 * one Ethernet device supported. The driver will need an
+	 * allocation scheme when VFs support Ethernet.
+	 */
+	dev->res.portal_index_indir_base = 0;
+
+	/* Enable some default hash type.
+	 * TODO: remove when ethtool supports them.
+	 */
+	dev->res.hash_types_enabled =
+		BIT(RSS_HASH_BIT(C_RSS_HASH_IPV4_TCP)) |
+		BIT(RSS_HASH_BIT(C_RSS_HASH_IPV4_UDP)) |
+		BIT(RSS_HASH_BIT(C_RSS_HASH_IPV4_PROTOCOL_UDP_ROCE)) |
+		BIT(RSS_HASH_BIT(C_RSS_HASH_IPV6_TCP)) |
+		BIT(RSS_HASH_BIT(C_RSS_HASH_IPV6_UDP)) |
+		BIT(RSS_HASH_BIT(C_RSS_HASH_IPV6_PROTOCOL_UDP_ROCE));
+
+	cxi_eth_cfg_timestamp(dev->cxi_dev, &ts_cfg);
+
+	return hw_setup(dev);
+}
+
+int cxi_eth_close(struct net_device *ndev)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	hw_cleanup(dev);
+
+	return 0;
+}
+
+/* Get a flow hash for the commands */
+static u8 get_flow_hash(struct sk_buff *skb)
+{
+	u32 hash = skb_get_hash(skb);
+
+	/* XOR the 4 bytes into one */
+	hash ^= hash >> 16;
+	hash ^= hash >> 8;
+
+	return hash & 0xff;
+}
+
+static void cxi_eth_get_skb_checksum_settings(struct sk_buff *skb,
+					      struct cxi_eth *dev,
+					      enum c_checksum_ctrl *ctrl,
+					      unsigned int *start,
+					      unsigned int *offset)
+{
+	__be16 proto;
+
+	if (skb->ip_summed != CHECKSUM_PARTIAL)
+		goto no_crc;
+
+	*start = skb_checksum_start_offset(skb) / 2;
+	*offset = skb->csum_offset / 2;
+
+	if (eth_type_vlan(skb->protocol))
+		proto = vlan_get_protocol(skb);
+	else
+		proto = skb->protocol;
+
+	if (proto == htons(ETH_P_IP)) {
+		struct iphdr *iph = ip_hdr(skb);
+
+		if (iph->protocol == IPPROTO_UDP) {
+			if ((dev->priv_flags & CXI_ETH_PF_ROCE_OPT) &&
+			    ntohs(udp_hdr(skb)->dest) == ROCE_V2_UDP_DPORT)
+				*ctrl = C_CHECKSUM_CTRL_ROCE;
+			else
+				*ctrl = C_CHECKSUM_CTRL_UDP;
+		} else if (iph->protocol == IPPROTO_TCP) {
+			*ctrl = C_CHECKSUM_CTRL_TCP;
+		}
+	} else if (proto == htons(ETH_P_IPV6)) {
+		struct ipv6hdr *iph = ipv6_hdr(skb);
+
+		if (iph->nexthdr == IPPROTO_UDP) {
+			if ((dev->priv_flags & CXI_ETH_PF_ROCE_OPT) &&
+			    ntohs(udp_hdr(skb)->dest) == ROCE_V2_UDP_DPORT)
+				*ctrl = C_CHECKSUM_CTRL_ROCE;
+			else
+				*ctrl = C_CHECKSUM_CTRL_UDP;
+		} else if (iph->nexthdr == IPPROTO_TCP) {
+			*ctrl = C_CHECKSUM_CTRL_TCP;
+		}
+	} else {
+		netdev_warn_once(dev->ndev,
+				 "Checksum type not supported. Protocol=%u\n",
+				 skb->protocol);
+		goto no_crc;
+	}
+
+	return;
+
+no_crc:
+	*ctrl = C_CHECKSUM_CTRL_NONE;
+	*start = 0;
+	*offset = 0;
+}
+
+/* Emit a packet using an IDC command */
+static netdev_tx_t emit_idc(struct sk_buff *skb, struct tx_queue *txq,
+			    struct cxi_cq *cq)
+{
+	struct c_idc_eth_cmd idc_eth = {
+		.fmt = C_PKT_FORMAT_STD,
+	};
+	struct cxi_eth *dev = txq->dev;
+	struct net_device *ndev = dev->ndev;
+	enum c_checksum_ctrl checksum_ctrl;
+	unsigned int checksum_start;
+	unsigned int checksum_offset;
+	int rc;
+
+	cxi_eth_get_skb_checksum_settings(skb, dev, &checksum_ctrl,
+					  &checksum_start, &checksum_offset);
+
+	idc_eth.checksum_ctrl = checksum_ctrl;
+	idc_eth.checksum_start = checksum_start;
+	idc_eth.checksum_offset = checksum_offset;
+	idc_eth.flow_hash = get_flow_hash(skb);
+
+	skb_tx_timestamp(skb);
+
+	rc = cxi_cq_emit_idc_eth(cq, &idc_eth, skb->data, skb->len);
+	if (rc) {
+		/* Hard error which should never happen. */
+		netdev_err(dev->ndev, "TX queue %u full when queue awake\n",
+			   txq->id);
+		txq->stats.tx_busy++;
+
+		return NETDEV_TX_BUSY;
+	}
+
+	txq->stats.idc++;
+	txq->stats.idc_bytes += skb->len;
+
+	ndev->stats.tx_packets++;
+	ndev->stats.tx_bytes += skb->len;
+
+	dev_kfree_skb(skb);
+
+	return NETDEV_TX_OK;
+}
+
+/* Emit a packet using a DMA command */
+static netdev_tx_t emit_dma(struct sk_buff *skb, struct tx_queue *txq,
+			    struct cxi_cq *cq, unsigned int *bytes)
+{
+	struct cxi_eth *dev = txq->dev;
+	struct device *dma_dev = &dev->cxi_dev->pdev->dev;
+	struct net_device *ndev = dev->ndev;
+	struct c_dma_eth_cmd dma_eth = {
+		.read_lac = txq->dev->phys_lac,
+		.fmt = C_PKT_FORMAT_STD,
+		.checksum_ctrl = C_CHECKSUM_CTRL_NONE,
+		.eq = txq->eq->eqn,
+		.user_ptr = (uintptr_t)skb,
+		.total_len = skb->len,
+	};
+	enum c_checksum_ctrl checksum_ctrl;
+	unsigned int checksum_start;
+	unsigned int checksum_offset;
+	int rc;
+
+	if (skb_shinfo(skb)->nr_frags >= C_MAX_ETH_FRAGS) {
+		if (skb_linearize(skb) ||
+		    skb_shinfo(skb)->nr_frags >= C_MAX_ETH_FRAGS) {
+			netdev_warn(ndev, "skb could not be linearized\n");
+			cancel_tx_tstamp(dev, skb);
+			dev_kfree_skb(skb);
+			ndev->stats.tx_dropped++;
+			return NETDEV_TX_OK;
+		}
+	}
+
+	if (prepare_frags(dma_dev, skb, &dma_eth)) {
+		netdev_warn(ndev, "skb could not be mapped for DMA\n");
+		cancel_tx_tstamp(dev, skb);
+		dev_kfree_skb(skb);
+		ndev->stats.tx_dropped++;
+		return NETDEV_TX_OK;
+	}
+
+	cxi_eth_get_skb_checksum_settings(skb, dev, &checksum_ctrl,
+					  &checksum_start, &checksum_offset);
+
+	dma_eth.checksum_ctrl = checksum_ctrl;
+	dma_eth.checksum_start = checksum_start;
+	dma_eth.checksum_offset = checksum_offset;
+	dma_eth.flow_hash = get_flow_hash(skb);
+
+	skb_tx_timestamp(skb);
+
+	rc = cxi_cq_emit_dma_eth(cq, &dma_eth);
+	if (rc) {
+		/* Hard error which should never happen. */
+		netdev_err(dev->ndev, "TX queue %u full when queue awake\n",
+			   txq->id);
+		txq->stats.tx_busy++;
+
+		unmap_frags(dma_dev, skb);
+		cancel_tx_tstamp(dev, skb);
+
+		return NETDEV_TX_BUSY;
+	}
+
+	atomic_dec(&txq->dma_eq_cdt);
+	txq->stats.dma++;
+	txq->stats.dma_bytes += skb->len;
+
+	*bytes += skb->len;
+	return NETDEV_TX_OK;
+}
+
+/* Check whether an outgoing packet can be timestamped, and reserve
+ * the unique slot if it can.
+ */
+static bool tstamp_xmit(struct cxi_eth *dev, struct sk_buff *skb)
+{
+	struct ethhdr *ethhdr;
+
+	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
+		return false;
+
+	/* Only Ethernet PTP packets are supported. That's what was
+	 * programmed in cxi_eth_cfg_timestamp().
+	 */
+	ethhdr = (struct ethhdr *)skb->data;
+	if (ethhdr->h_proto != htons(PTP_L2_ETHERTYPE))
+		return false;
+
+	if (cmpxchg(&dev->tstamp_skb, NULL, skb))
+		return false;
+
+	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+
+	return true;
+}
+
+static void ring_txq(struct tx_queue *txq)
+{
+	cxi_cq_ring(txq->eth1_cq);
+	if (txq->eth2_cq)
+		cxi_cq_ring(txq->eth2_cq);
+	cxi_cq_ring(txq->shared_cq);
+}
+
+netdev_tx_t cxi_eth_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	unsigned int qid = skb_get_queue_mapping(skb);
+	struct tx_queue *txq = &dev->txqs[qid];
+	struct netdev_queue *dev_queue =
+		netdev_get_tx_queue(dev->ndev, txq->id);
+	unsigned int bytes = 0;
+	netdev_tx_t tx_status;
+	bool kick_cq;
+	bool more = netdev_xmit_more();
+	struct cxi_cq *cq;
+
+	if (eth_type_vlan(skb->protocol)) {
+		/* Tagged traffic. Check PCPs to steer appropriately */
+		struct vlan_ethhdr *veth;
+		unsigned int pcp;
+
+		veth = (struct vlan_ethhdr *)vlan_eth_hdr(skb);
+		pcp = (be16_to_cpu(veth->h_vlan_TCI) & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
+
+		if (pcp == dev->cxi_dev->untagged_eth_pcp)
+			cq = txq->untagged_cq;
+		else if (pcp == dev->eth1_pcp)
+			cq = txq->eth1_cq;
+		else if (dev->eth2_active && pcp == dev->eth2_pcp)
+			cq = txq->eth2_cq;
+		else
+			cq = txq->shared_cq;
+	} else {
+		/* Untagged traffic. */
+		cq = txq->untagged_cq;
+	}
+
+	if (tstamp_xmit(dev, skb) || skb_shinfo(skb)->nr_frags ||
+	    skb->len > idc_dma_threshold) {
+		tx_status = emit_dma(skb, txq, cq, &bytes);
+		txq->force_dma = txq->force_dma_interval;
+	} else if (--txq->force_dma == 0) {
+		/* Too many consecutive IDC. Use DMA instead */
+		tx_status = emit_dma(skb, txq, cq, &bytes);
+		txq->force_dma = txq->force_dma_interval;
+		txq->stats.dma_forced++;
+	} else {
+		tx_status = emit_idc(skb, txq, cq);
+	}
+
+	/* Kick CQ if netdev TX queue is ready. */
+	kick_cq = __netdev_tx_sent_queue(dev_queue, bytes, more);
+	if (kick_cq)
+		ring_txq(txq);
+
+	/* Stop TX queue if not enough free slots to emit another SKB
+	 * or NETDEV_TX_BUSY is set. 8 slots, aligned on an 8-slot
+	 * boundary, are required to emit the largest IDC or DMA
+	 * commands. TX queue will be re-enabled as space frees up. If
+	 * the TX queue is to be stopped, the TX queue is always
+	 * kicked in an attempt to free up space.
+	 */
+	if (__cxi_cq_free_slots(cq) < 16 || tx_status == NETDEV_TX_BUSY) {
+		if (!kick_cq)
+			ring_txq(txq);
+		netif_tx_stop_queue(dev_queue);
+		txq->stats.stopped++;
+	} else if (atomic_read(&txq->dma_eq_cdt) <= 0) {
+		/* The EQ is getting full */
+		if (!kick_cq)
+			ring_txq(txq);
+		netif_tx_stop_queue(dev_queue);
+		txq->stats.stopped_eq++;
+	}
+
+	return tx_status;
+}
+
+int cxi_eth_mac_addr(struct net_device *ndev, void *p)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	int rc;
+
+	rc = eth_mac_addr(ndev, p);
+	if (rc)
+		return rc;
+
+	dev->mac_addr = ether_addr_to_u64(ndev->dev_addr);
+	ether_addr_copy(dev->cxi_dev->mac_addr, ndev->dev_addr);
+
+	cxi_eth_set_rx_mode(ndev);
+
+	/* We expect the NID to be configured before the MAC address is set.
+	 * To be backwards compatible, we will attempt to set the NID from the MAC address.
+	 * If the NID was not set, it will be derived from the MAC. Otherwise this will be ignored
+	 */
+	cxi_set_nid_from_mac(dev->cxi_dev, ndev->dev_addr);
+
+	return 0;
+}
+
+/* Program MAC filters.
+ *
+ * Since the ethernet driver is the only thing using the MAC filter
+ * (set_list), we can reset the whole list when entering this
+ * function, and reprogram everything.
+ */
+void cxi_eth_set_rx_mode(struct net_device *ndev)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct netdev_hw_addr *ha;
+	int rc;
+
+	/* Invalidate all existing filters. */
+	cxi_eth_set_list_invalidate_all(dev->cxi_dev, &dev->res);
+
+	/* Cassini ERRATA-3258. Workaround timestamp bug. Program the PTP MAC
+	 * address first so all the PTP packets will land on the PTP
+	 * RX queue, even if promiscuous mode is enabled.
+	 */
+	cxi_eth_add_mac(dev->cxi_dev, &dev->res, dev->ptp_mac_addr, true);
+
+	if (ndev->flags & IFF_PROMISC) {
+		/* Listen on everything. */
+		cxi_eth_set_promiscuous(dev->cxi_dev, &dev->res);
+		cxi_eth_set_indir_table(dev->cxi_dev, &dev->res);
+		return;
+	}
+
+	rc = cxi_eth_add_mac(dev->cxi_dev, &dev->res, dev->mac_addr, false);
+	if (rc) {
+		netdev_err(ndev, "Cannot program MAC address\n");
+		return;
+	}
+
+	if (ndev->flags & IFF_BROADCAST) {
+		rc = cxi_eth_add_mac(dev->cxi_dev, &dev->res,
+				     0xffffffffffffULL, false);
+		if (rc) {
+			netdev_err(ndev, "Cannot program broadcast address\n");
+			return;
+		}
+	}
+
+	netdev_for_each_uc_addr(ha, ndev) {
+		netdev_dbg(ndev, "Adding UC addr %pM\n", ha->addr);
+
+		rc = cxi_eth_add_mac(dev->cxi_dev, &dev->res,
+				     ether_addr_to_u64(ha->addr), false);
+		if (rc) {
+			netdev_err(ndev, "Cannot program unicast address\n");
+			return;
+		}
+	}
+
+	if (ndev->flags & IFF_ALLMULTI) {
+		/* Accept all multicasts. */
+		rc = cxi_eth_set_all_multi(dev->cxi_dev, &dev->res);
+		if (rc) {
+			netdev_err(ndev,
+				   "Cannot program all multicast addresses\n");
+			return;
+		}
+	} else if (ndev->flags & IFF_MULTICAST) {
+		netdev_for_each_mc_addr(ha, ndev) {
+			netdev_dbg(ndev, "Adding MC addr %pM\n", ha->addr);
+
+			rc = cxi_eth_add_mac(dev->cxi_dev, &dev->res,
+					     ether_addr_to_u64(ha->addr),
+					     false);
+			if (rc) {
+				netdev_err(ndev,
+					   "Cannot program multicast address\n");
+				return;
+			}
+		}
+	}
+
+	cxi_eth_set_indir_table(dev->cxi_dev, &dev->res);
+}
+
+int cxi_change_mtu(struct net_device *ndev, int new_mtu)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	bool was_running;
+	int rc;
+
+	was_running = netif_running(ndev);
+	if (was_running)
+		hw_cleanup(dev);
+
+	rc = cxi_set_max_eth_rxsize(dev->cxi_dev, new_mtu + VLAN_ETH_HLEN);
+	if (rc)
+		netdev_err(ndev, "Invalid MTU size: %u\n", new_mtu);
+	else
+		ndev->mtu = new_mtu;
+
+	if (was_running) {
+		int rc2;
+
+		rc2 = hw_setup(dev);
+		if (rc2) {
+			netdev_err(ndev,
+				   "Failed to setup resources after an MTU change\n");
+			rc = rc2;
+		}
+	}
+
+	return rc;
+}

@@ -0,0 +1,916 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cassini ethernet driver
+ * Copyright 2018,2022,2024,2026 Hewlett Packard Enterprise Development LP
+ */
+
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/firmware.h>
+#include <uapi/linux/net_tstamp.h>
+
+#include "cxi_eth.h"
+#include "cxi_link.h"
+/*
+ * definitions for CXI_GLOBAL_STATS_LEN, cxi_get_ethtool_stats_set,
+ * and cxi_get_ethtool_stats_name.
+ */
+#include "cxi_ethtool_names.h"
+
+/* Ethtool priv_flags */
+static const char priv_flags_str[PRIV_FLAGS_COUNT][ETH_GSTRING_LEN] = {
+	"internal-loopback",
+	"external-loopback",
+	"llr",
+	"precoding",
+	"ifg-hpc",
+	"roce-opt",
+	"ignore-align",
+	"pml-recovery",
+	"link-train",
+	"remote-fault-recovery",
+	"use-unsupported-cable",
+	"fec_monitor",
+	"auto-lane-degrade",
+	"ignore-media-error",
+	"use-supported-ss200-cable",
+	"los-lol-hide",
+	"r1-link-partner"
+};
+
+/* ethtool ops */
+static void cxi_get_drvinfo(struct net_device *ndev,
+			    struct ethtool_drvinfo *info)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	strscpy(info->driver, "cxi_eth", sizeof(info->driver));
+	strscpy(info->version, "0.2", sizeof(info->version));
+	strscpy(info->bus_info, dev_name(ndev->dev.parent),
+		sizeof(info->bus_info));
+	strscpy(info->fw_version, dev->eth_info.fw_version,
+		sizeof(info->fw_version));
+	strscpy(info->erom_version, dev->eth_info.erom_version,
+		sizeof(info->erom_version));
+
+	info->n_priv_flags = PRIV_FLAGS_COUNT;
+}
+
+static int cxi_get_sset_count(struct net_device *ndev, int sset)
+{
+	switch (sset) {
+	case ETH_SS_PRIV_FLAGS:
+		return PRIV_FLAGS_COUNT;
+	case ETH_SS_STATS:
+		return CXI_GLOBAL_STATS_LEN;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void cxi_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
+{
+	char *p = data;
+	struct cxi_eth *dev = netdev_priv(ndev);
+	char *dst;
+
+	switch (stringset) {
+	case ETH_SS_PRIV_FLAGS:
+		memcpy(p, priv_flags_str, PRIV_FLAGS_COUNT * ETH_GSTRING_LEN);
+		if (cassini_version(&dev->cxi_dev->prop, CASSINI_1)) {
+			dst = p + 7 * ETH_GSTRING_LEN;
+			memset(dst, 0, ETH_GSTRING_LEN);
+			strscpy(dst, "disable-pml-recovery", ETH_GSTRING_LEN);
+		}
+		break;
+	case ETH_SS_STATS:
+		memcpy(p, cxi_get_ethtool_stats_name, CXI_GLOBAL_STATS_LEN *
+		       ETH_GSTRING_LEN);
+		break;
+
+	}
+}
+
+/* Return the number of RSS queues. */
+static void cxi_get_channels(struct net_device *ndev,
+			     struct ethtool_channels *channels)
+{
+	channels->max_rx = max_rss_queues;
+	channels->max_tx = max_tx_queues;
+	channels->max_combined = 0;
+	channels->max_other = 0;
+	channels->rx_count = ndev->real_num_rx_queues;
+	channels->tx_count = ndev->real_num_tx_queues;
+	channels->other_count = 0;
+	channels->combined_count = 0;
+}
+
+static int cxi_grow_tx_channels(struct cxi_eth *dev,
+				unsigned int num_tx_channels)
+{
+	int i;
+	int rc;
+
+	for (i = dev->cur_txqs; i < num_tx_channels; i++) {
+		rc = alloc_tx_queue(dev, i);
+		if (rc)
+			goto err_free_txqs;
+	}
+
+	for (i = dev->cur_txqs; i < num_tx_channels; i++)
+		enable_tx_queue(&dev->txqs[i]);
+
+	rc = netif_set_real_num_tx_queues(dev->ndev, num_tx_channels);
+	if (rc)
+		goto err_disable_txqs;
+
+	return 0;
+
+err_disable_txqs:
+	for (i = dev->cur_txqs; i < num_tx_channels; i++)
+		disable_tx_queue(&dev->txqs[i]);
+err_free_txqs:
+	while (i > dev->cur_txqs)
+		free_tx_queue(&dev->txqs[--i]);
+
+	return rc;
+}
+
+static int cxi_shrink_tx_channels(struct cxi_eth *dev,
+				  unsigned int num_tx_channels)
+{
+	int i;
+	int rc;
+
+	rc = netif_set_real_num_tx_queues(dev->ndev, num_tx_channels);
+	if (rc)
+		return rc;
+
+	for (i = dev->cur_txqs - 1; i >= num_tx_channels; i--)
+		disable_tx_queue(&dev->txqs[i]);
+
+	for (i = dev->cur_txqs - 1; i >= num_tx_channels; i--)
+		free_tx_queue(&dev->txqs[i]);
+
+	return 0;
+}
+
+int cxi_set_tx_channels(struct cxi_eth *dev, unsigned int num_tx_channels)
+{
+	int rc;
+
+	if (dev->cur_txqs == num_tx_channels)
+		return 0;
+	else if (num_tx_channels < dev->cur_txqs)
+		rc = cxi_shrink_tx_channels(dev, num_tx_channels);
+	else
+		rc = cxi_grow_tx_channels(dev, num_tx_channels);
+
+	if (!rc)
+		dev->cur_txqs = num_tx_channels;
+
+	return rc;
+}
+
+static int cxi_grow_rx_channels(struct cxi_eth *dev,
+				unsigned int num_rx_channels)
+{
+	int rc;
+	int i;
+
+	for (i = dev->res.rss_queues; i < num_rx_channels; i++) {
+		rc = alloc_rx_queue(dev, i);
+		if (rc)
+			goto err_free_rxqs;
+
+		rc = post_rx_buffers(&dev->rxqs[i], GFP_KERNEL);
+		if (rc < 0) {
+			netdev_info(dev->ndev, "Cannot post RX buffers: %d\n",
+				    rc);
+			/* Increment i to free this RXQ. */
+			i++;
+			goto err_free_rxqs;
+		}
+	}
+
+	for (i = dev->res.rss_queues; i < num_rx_channels; i++)
+		enable_rx_queue(&dev->rxqs[i]);
+
+	return 0;
+
+err_free_rxqs:
+	for (i--; i >= dev->res.rss_queues; i--)
+		free_rx_queue(&dev->rxqs[i]);
+
+	return rc;
+}
+
+static void cxi_shrink_rx_channels(struct cxi_eth *dev,
+				   unsigned int num_rx_channels)
+{
+	int i;
+
+	for (i = dev->res.rss_queues - 1; i >= num_rx_channels; i--)
+		disable_rx_queue(&dev->rxqs[i]);
+
+	for (i = dev->res.rss_queues - 1; i >= num_rx_channels; i--)
+		free_rx_queue(&dev->rxqs[i]);
+}
+
+static bool valid_rx_channel_count(unsigned int num_rx_channels)
+{
+	if (!is_power_of_2(num_rx_channels) || num_rx_channels < 1 ||
+	    num_rx_channels > max_rss_queues)
+		return false;
+	return true;
+}
+
+int cxi_set_rx_channels(struct cxi_eth *dev, unsigned int num_rx_channels)
+{
+	unsigned int old_rx_channels = dev->ndev->real_num_rx_queues;
+	int rc;
+	int i;
+
+	if (!valid_rx_channel_count(num_rx_channels))
+		return -EINVAL;
+
+	if (num_rx_channels == dev->res.rss_queues)
+		return 0;
+
+	/* Disable RSS support for all configure MAC addresses/set lists. This
+	 * will temporarily force all traffic to the RXQ 0 which is the
+	 * default/catch-all RX queue.
+	 */
+	cxi_eth_clear_indir_table(dev->cxi_dev, &dev->res);
+
+	rc = netif_set_real_num_rx_queues(dev->ndev, num_rx_channels);
+	if (rc) {
+		cxi_eth_set_indir_table(dev->cxi_dev, &dev->res);
+		goto out;
+	}
+
+	if (num_rx_channels > dev->res.rss_queues) {
+		rc = cxi_grow_rx_channels(dev, num_rx_channels);
+		if (rc) {
+			netif_set_real_num_rx_queues(dev->ndev, old_rx_channels);
+			/* Number of channels have not changed. Re-enable RSS
+			 * and exit.
+			 */
+			cxi_eth_set_indir_table(dev->cxi_dev, &dev->res);
+			goto out;
+		}
+	} else {
+		cxi_shrink_rx_channels(dev, num_rx_channels);
+	}
+
+	dev->res.rss_queues = num_rx_channels;
+	dev->res.rss_indir_size = num_rx_channels > 1 ? rss_indir_size : 0;
+
+	for (i = 0; i < dev->res.rss_queues; i++)
+		dev->res.ptn_rss[i] = dev->rxqs[i].pt->id;
+
+	for (i = 0; i < dev->res.rss_indir_size; i++)
+		dev->res.indir_table[i] =
+			ethtool_rxfh_indir_default(i, dev->res.rss_queues);
+
+	/* Re-enable RSS if number of RX queues is successfully changed. */
+	cxi_eth_set_indir_table(dev->cxi_dev, &dev->res);
+
+out:
+	return rc;
+}
+
+static int cxi_set_channels(struct net_device *ndev,
+			    struct ethtool_channels *ch)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	unsigned int old_tx_channels = dev->cur_txqs;
+	int rc;
+
+	if (ch->combined_count || ch->other_count)
+		return -EINVAL;
+	if (!valid_rx_channel_count(ch->rx_count) || !ch->tx_count ||
+	    ch->tx_count > ndev->num_tx_queues)
+		return -EINVAL;
+
+	/* If interface is not running, update the real TX/RX queue count. The
+	 * real TX/RX queue count is used to initialize TX/RX channels during
+	 * hardware setup.
+	 */
+	if (!netif_running(dev->ndev)) {
+		return netif_set_real_num_queues(ndev, ch->tx_count, ch->rx_count);
+	}
+
+	if (ch->tx_count > old_tx_channels) {
+		rc = cxi_set_tx_channels(dev, ch->tx_count);
+		if (rc)
+			return rc;
+	}
+
+	rc = cxi_set_rx_channels(dev, ch->rx_count);
+	if (rc) {
+		if (ch->tx_count > old_tx_channels)
+			WARN_ON_ONCE(cxi_set_tx_channels(dev, old_tx_channels));
+		return rc;
+	}
+
+	if (ch->tx_count < old_tx_channels)
+		return cxi_set_tx_channels(dev, ch->tx_count);
+	return 0;
+}
+
+static int cxi_get_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *cmd,
+			 u32 *rule_locs)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	int rc;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_GRXRINGS:
+		/* The catch-all queue is always present */
+		cmd->data = dev->res.rss_queues ? : 1;
+		rc = 0;
+		break;
+
+	default:
+		rc = -EOPNOTSUPP;
+		break;
+	}
+
+	return rc;
+}
+
+static u32 cxi_get_rxfh_indir_size(struct net_device *ndev)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	return dev->res.rss_indir_size;
+}
+
+static u32 cxi_get_rxfh_key_size(struct net_device *ndev)
+{
+	return CXI_ETH_HASH_KEY_SIZE;
+}
+
+static int cxi_get_rxfh(struct net_device *ndev,
+			struct ethtool_rxfh_param *rxfh)
+{
+	u8 *key = rxfh->key;
+	u8 *hfunc = &rxfh->hfunc;
+	u32 *indir = rxfh->indir;
+
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	if (hfunc)
+		*hfunc = ETH_RSS_HASH_TOP;
+
+	if (key)
+		cxi_eth_get_hash_key(dev->cxi_dev, key);
+
+	if (indir && dev->res.rss_queues)
+		memcpy(indir, dev->res.indir_table,
+		       dev->res.rss_indir_size * sizeof(u32));
+
+	return 0;
+}
+
+static int cxi_set_rxfh(struct net_device *ndev,
+			struct ethtool_rxfh_param *rxfh,
+			struct netlink_ext_ack *extack)
+{
+	u8 *key = rxfh->key;
+	u8 hfunc = rxfh->hfunc;
+	u32 *indir = rxfh->indir;
+
+	struct cxi_eth *dev = netdev_priv(ndev);
+	int i;
+
+	if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
+		return -EINVAL;
+
+	if (key != NULL)
+		return -EINVAL;
+
+	if (indir) {
+		/* Sanity check the table */
+		for (i = 0; i < dev->res.rss_indir_size; i++) {
+			if (indir[i] >= dev->res.rss_queues) {
+				netdev_warn_once(ndev, "Bad new indirection - %d %d\n",
+						 indir[i],
+						 dev->res.rss_queues);
+				return -EINVAL;
+			}
+		}
+
+		memcpy(dev->res.indir_table, indir,
+		       dev->res.rss_indir_size * sizeof(u32));
+		cxi_eth_set_indir_table(dev->cxi_dev, &dev->res);
+	}
+
+	return 0;
+}
+
+static int cxi_get_module_info(struct net_device *ndev,
+			       struct ethtool_modinfo *modinfo)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	cxi_eth_devinfo(dev->cxi_dev, &dev->eth_info);
+
+	if (dev->eth_info.qsfp_eeprom_len == 0)
+		return -ENOMEDIUM;
+
+	switch (dev->eth_info.qsfp_eeprom[0]) {
+	case 0xc: /* QSFP */
+	case 0xd: /* QSFP+ */
+		modinfo->type = ETH_MODULE_SFF_8436;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN;
+		break;
+	case 0x11: /* QSFP-28 */
+	case 0x1E:
+		modinfo->type = ETH_MODULE_SFF_8636;
+		modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN;
+		break;
+	default:
+		netdev_err(dev->ndev, "Unsupported transceiver type 0x%x\n",
+			   dev->eth_info.qsfp_eeprom[0]);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int cxi_get_module_eeprom(struct net_device *ndev,
+				 struct ethtool_eeprom *eeprom, u8 *data)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	int ret;
+
+	ret = cxi_get_qsfp_data(dev->cxi_dev, eeprom->offset, eeprom->len,
+				0, data);
+
+	if (ret > 0)
+		return 0;
+	else
+		return -EINVAL;
+}
+
+static int cxi_get_module_eeprom_by_page(struct net_device *ndev,
+					 const struct ethtool_module_eeprom *req,
+					 struct netlink_ext_ack *extack)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	return cxi_get_qsfp_data(dev->cxi_dev, req->offset, req->length,
+				 req->page, req->data);
+}
+
+static int cxi_flash_device(struct net_device *ndev,
+			    struct ethtool_flash *flash)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	const struct firmware *fw;
+	int rc;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	rc = request_firmware(&fw, flash->data, &ndev->dev);
+	if (rc != 0)
+		return rc;
+
+	// TODO? stop adapter
+	rc = cxi_program_firmware(dev->cxi_dev, fw);
+	release_firmware(fw);
+	// TODO? restart adapter
+
+	if (!rc)
+		cxi_eth_devinfo(dev->cxi_dev, &dev->eth_info);
+
+	return rc;
+}
+
+/* Link settings. */
+static int cxi_get_link_ksettings(struct net_device *ndev,
+				  struct ethtool_link_ksettings *s)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct cxi_link_info link_info;
+
+	memset(s, 0, sizeof(struct ethtool_link_ksettings));
+
+	cxi_link_mode_get(dev->cxi_dev, &link_info);
+
+	s->base.speed = link_info.speed;
+	s->base.port = link_info.port_type;
+	s->base.duplex = DUPLEX_FULL;
+	s->base.phy_address = 0; /* N/A */
+	s->base.autoneg = link_info.autoneg;
+	s->base.eth_tp_mdix = ETH_TP_MDI;
+	s->base.eth_tp_mdix_ctrl = ETH_TP_MDI;
+
+	ethtool_link_ksettings_add_link_mode(s, supported, Autoneg);
+	ethtool_link_ksettings_add_link_mode(s, supported, FIBRE);
+	ethtool_link_ksettings_add_link_mode(s, supported, TP);
+	ethtool_link_ksettings_add_link_mode(s, supported, Backplane);
+	ethtool_link_ksettings_add_link_mode(s, supported, Pause);
+	if (cassini_version(&dev->cxi_dev->prop, CASSINI_1)) {
+		ethtool_link_ksettings_add_link_mode(s, supported, 50000baseKR_Full);
+		ethtool_link_ksettings_add_link_mode(s, supported, 100000baseCR2_Full);
+		ethtool_link_ksettings_add_link_mode(s, supported, 100000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(s, supported, 200000baseCR4_Full);
+		if (s->base.autoneg == AUTONEG_ENABLE)
+			ethtool_link_ksettings_add_link_mode(s, advertising, Autoneg);
+		ethtool_link_ksettings_add_link_mode(s, advertising, Pause);
+		ethtool_link_ksettings_add_link_mode(s, advertising, 50000baseKR_Full);
+		ethtool_link_ksettings_add_link_mode(s, advertising, 100000baseCR2_Full);
+		ethtool_link_ksettings_add_link_mode(s, advertising, 100000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(s, advertising, 200000baseCR4_Full);
+	} else if (cassini_version(&dev->cxi_dev->prop, CASSINI_2)) {
+		ethtool_link_ksettings_add_link_mode(s, supported, 100000baseCR2_Full);
+		ethtool_link_ksettings_add_link_mode(s, supported, 200000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(s, supported, 400000baseCR4_Full);
+		ethtool_link_ksettings_add_link_mode(s, supported, FEC_RS);
+		if (s->base.autoneg == AUTONEG_ENABLE) {
+			ethtool_link_ksettings_add_link_mode(s, advertising, Autoneg);
+			ethtool_link_ksettings_add_link_mode(s, advertising, 100000baseCR2_Full);
+			ethtool_link_ksettings_add_link_mode(s, advertising, 200000baseCR4_Full);
+			ethtool_link_ksettings_add_link_mode(s, advertising, 400000baseCR4_Full);
+			ethtool_link_ksettings_add_link_mode(s, advertising, FEC_RS);
+		}
+	}
+
+	return 0;
+}
+
+static int
+cxi_set_link_ksettings(struct net_device *ndev,
+		       const struct ethtool_link_ksettings *s)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct cxi_link_info link_info;
+
+	cxi_link_mode_get(dev->cxi_dev, &link_info);
+
+	if (s->base.duplex != DUPLEX_UNKNOWN &&
+	    s->base.duplex != DUPLEX_FULL)
+		return -ENOTSUPP;
+
+	if (s->base.eth_tp_mdix != ETH_TP_MDI ||
+	    s->base.eth_tp_mdix_ctrl != ETH_TP_MDI)
+		return -ENOTSUPP;
+
+	if (dev->is_c2 && s->base.autoneg == AUTONEG_ENABLE &&
+	    (link_info.flags & LOOPBACK_MODE) != 0) {
+		netdev_err(ndev,
+			   "autoneg must not be enabled in loopback mode\n");
+		return -EINVAL;
+	}
+
+	link_info.speed = s->base.speed;
+	link_info.autoneg = s->base.autoneg;
+
+	cxi_link_mode_set(dev->cxi_dev, &link_info);
+
+	return 0;
+}
+
+static int cxi_set_priv_flags(struct net_device *ndev, u32 flags)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct cxi_link_info link_info;
+	u32 changes;
+	u32 old_flags;
+	u32 debug_flags;
+
+	cxi_link_mode_get(dev->cxi_dev, &link_info);
+	cxi_link_flags_get(dev->cxi_dev, &debug_flags);
+
+	old_flags = dev->priv_flags | link_info.flags | debug_flags;
+
+	changes = flags ^ old_flags;
+
+	if (!changes)
+		return 0;
+
+	if (changes & LOOPBACK_MODE) {
+		u32 loopback_mode = flags & LOOPBACK_MODE;
+
+		if (loopback_mode ==
+		    (CXI_ETH_PF_INTERNAL_LOOPBACK | CXI_ETH_PF_EXTERNAL_LOOPBACK)) {
+			netdev_err(dev->ndev,
+				   "Loopback private flags are mutually exclusive\n");
+			return -EINVAL;
+		}
+	}
+
+	if (changes & flags & LINK_DEBUG_FLAGS &
+	    ~cxi_link_flags_supported(dev->cxi_dev)) {
+		netdev_err(ndev, "Link debug private flag not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	link_info.flags = flags;
+	cxi_link_mode_set(dev->cxi_dev, &link_info);
+
+	if (changes & CXI_ETH_PF_FEC_MONITOR) {
+		if (flags & CXI_ETH_PF_FEC_MONITOR)
+			dev->priv_flags |= CXI_ETH_PF_FEC_MONITOR;
+		else
+			dev->priv_flags &= ~CXI_ETH_PF_FEC_MONITOR;
+	}
+
+	if (changes & CXI_ETH_PF_ROCE_OPT) {
+		if (flags & CXI_ETH_PF_ROCE_OPT)
+			dev->priv_flags |= CXI_ETH_PF_ROCE_OPT;
+		else
+			dev->priv_flags &= ~CXI_ETH_PF_ROCE_OPT;
+
+		cxi_set_roce_rcv_seg(dev->cxi_dev, flags & CXI_ETH_PF_ROCE_OPT);
+	}
+
+	if (changes & CXI_ETH_PF_IGNORE_ALIGN) {
+		if (flags & CXI_ETH_PF_IGNORE_ALIGN)
+			cxi_link_flags_set(dev->cxi_dev,
+					   0, CXI_ETH_PF_IGNORE_ALIGN);
+		else
+			cxi_link_flags_set(dev->cxi_dev,
+					   CXI_ETH_PF_IGNORE_ALIGN, 0);
+	}
+
+	if (changes & CXI_ETH_PF_PML_REC) {
+		if (flags & CXI_ETH_PF_PML_REC) {
+			dev->priv_flags |= CXI_ETH_PF_PML_REC;
+			cxi_pml_recovery_set(dev->cxi_dev, true);
+		} else {
+			dev->priv_flags &= ~CXI_ETH_PF_PML_REC;
+			cxi_pml_recovery_set(dev->cxi_dev, false);
+		}
+	}
+
+	if (changes & CXI_ETH_PF_REMOTE_FAULT_RECOVERY) {
+		if (flags & CXI_ETH_PF_REMOTE_FAULT_RECOVERY)
+			cxi_link_flags_set(dev->cxi_dev,
+					   0, CXI_ETH_PF_REMOTE_FAULT_RECOVERY);
+		else
+			cxi_link_flags_set(dev->cxi_dev,
+					   CXI_ETH_PF_REMOTE_FAULT_RECOVERY, 0);
+	}
+
+	if (changes & CXI_ETH_PF_USE_UNSUPPORTED_CABLE) {
+		if (flags & CXI_ETH_PF_USE_UNSUPPORTED_CABLE) {
+			dev->priv_flags |= CXI_ETH_PF_USE_UNSUPPORTED_CABLE;
+			cxi_link_use_unsupported_cable(dev->cxi_dev, true);
+		} else {
+			dev->priv_flags &= ~CXI_ETH_PF_USE_UNSUPPORTED_CABLE;
+			cxi_link_use_unsupported_cable(dev->cxi_dev, false);
+		}
+	}
+
+	if (changes & CXI_ETH_PF_USE_SUPPORTED_SS200_CABLE) {
+		if (flags & CXI_ETH_PF_USE_SUPPORTED_SS200_CABLE) {
+			dev->priv_flags |= CXI_ETH_PF_USE_SUPPORTED_SS200_CABLE;
+			cxi_link_use_supported_ss200_cable(dev->cxi_dev, true);
+		} else {
+			dev->priv_flags &= ~CXI_ETH_PF_USE_SUPPORTED_SS200_CABLE;
+			cxi_link_use_supported_ss200_cable(dev->cxi_dev, false);
+		}
+	}
+
+	if (changes & CXI_ETH_PF_IGNORE_MEDIA_ERROR) {
+		if (flags & CXI_ETH_PF_IGNORE_MEDIA_ERROR) {
+			dev->priv_flags |= CXI_ETH_PF_IGNORE_MEDIA_ERROR;
+			cxi_link_ignore_media_error(dev->cxi_dev, true);
+		} else {
+			dev->priv_flags &= ~CXI_ETH_PF_IGNORE_MEDIA_ERROR;
+			cxi_link_ignore_media_error(dev->cxi_dev, false);
+		}
+	}
+
+	if (changes & CXI_ETH_PF_ALD) {
+		if (flags & CXI_ETH_PF_ALD) {
+			dev->priv_flags |= CXI_ETH_PF_ALD;
+			cxi_link_auto_lane_degrade(dev->cxi_dev, true);
+		} else {
+			dev->priv_flags &= ~CXI_ETH_PF_ALD;
+			cxi_link_auto_lane_degrade(dev->cxi_dev, false);
+		}
+	}
+
+	if (changes & CXI_ETH_PF_LOS_LOL_HIDE) {
+		if (flags & CXI_ETH_PF_LOS_LOL_HIDE) {
+			dev->priv_flags |= CXI_ETH_PF_LOS_LOL_HIDE;
+			cxi_link_los_lol_hide(dev->cxi_dev, true);
+		} else {
+			dev->priv_flags &= ~CXI_ETH_PF_LOS_LOL_HIDE;
+			cxi_link_los_lol_hide(dev->cxi_dev, false);
+		}
+	}
+
+	if (changes & CXI_ETH_PF_R1_LINK_PARTNER) {
+		if (flags & CXI_ETH_PF_R1_LINK_PARTNER)
+			dev->priv_flags |= CXI_ETH_PF_R1_LINK_PARTNER;
+		else
+			dev->priv_flags &= ~CXI_ETH_PF_R1_LINK_PARTNER;
+	}
+
+	return 0;
+}
+
+static u32 cxi_get_priv_flags(struct net_device *ndev)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct cxi_link_info link_info;
+	u32 debug_flags;
+
+	cxi_link_mode_get(dev->cxi_dev, &link_info);
+	cxi_link_flags_get(dev->cxi_dev, &debug_flags);
+
+	return dev->priv_flags | link_info.flags | debug_flags;
+}
+
+static void cxi_get_ethtool_stats(struct net_device *ndev,
+				  struct ethtool_stats *stats, u64 *data)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct cxi_dev *cdev = dev->cxi_dev;
+	int retval;
+	int i;
+
+	retval = cxi_telem_get_selected(cdev, cxi_get_ethtool_stats_set, data,
+					CXI_GLOBAL_STATS_LEN);
+	if (retval != 0)
+		for (i = 0 ; i < CXI_GLOBAL_STATS_LEN ; ++i)
+			data[i] = 0;
+}
+
+static int cxi_get_ts_info(struct net_device *ndev,
+			   struct kernel_ethtool_ts_info *info)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	info->so_timestamping =
+		SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE |
+		SOF_TIMESTAMPING_TX_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE |
+		SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RAW_HARDWARE;
+
+	info->phc_index = dev->eth_info.ptp_clock_index;
+
+	info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON);
+
+	/* The HW only supports PTP Ethernet filter. Timestamp cannot
+	 * be added to PTP UDP packet.
+	 */
+	info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
+			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT);
+
+	return 0;
+}
+
+static void cxi_get_pauseparam(struct net_device *ndev,
+			       struct ethtool_pauseparam *pause)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	cxi_eth_get_pause(dev->cxi_dev, pause);
+	pause->autoneg = 0;
+}
+
+static int cxi_set_pauseparam(struct net_device *ndev,
+			      struct ethtool_pauseparam *pause)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	if (pause->autoneg != 0)
+		return -EINVAL;
+
+	cxi_eth_set_pause(dev->cxi_dev, pause);
+
+	return 0;
+}
+
+static int cxi_get_fecparam(struct net_device *ndev,
+			    struct ethtool_fecparam *fec)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	struct cxi_link_info link_info;
+
+	cxi_link_mode_get(dev->cxi_dev, &link_info);
+
+	fec->active_fec = link_info.fec_type;
+
+	return 0;
+}
+
+static int cxi_set_fecparam(struct net_device *ndev,
+			    struct ethtool_fecparam *fec)
+{
+	return -ENOTSUPP;
+}
+
+static void cxi_get_ringparam(struct net_device *ndev,
+			      struct ethtool_ringparam *ring,
+			      struct kernel_ethtool_ringparam *kernel_ring,
+			      struct netlink_ext_ack *extack)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	*ring = dev->ringparam;
+
+#ifdef ETHTOOL_RING_USE_CQE_SIZE
+	kernel_ring->cqe_size = get_rxq_eq_buf_size(dev) /
+		sizeof(struct c_event_target_enet);
+#endif
+	kernel_ring->rx_buf_len =
+		dev->ringparam.rx_pending * dev->eth_info.max_segment_size +
+		small_pkts_buf_count * small_pkts_buf_size;
+}
+
+static int cxi_set_ringparam(struct net_device *ndev,
+			     struct ethtool_ringparam *ring,
+			     struct kernel_ethtool_ringparam *kernel_ring,
+			     struct netlink_ext_ack *extack)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+	int rc = 0;
+
+	if (ring->rx_jumbo_pending)
+		return -EINVAL;
+
+	dev->ringparam.rx_pending = clamp(ring->rx_pending,
+					  LARGE_PKTS_BUF_COUNT_MIN,
+					  LARGE_PKTS_BUF_COUNT_MAX);
+
+	dev->ringparam.rx_mini_pending = num_small_packets(
+		clamp(round_up(ring->rx_mini_pending, 4096) /
+		      (small_pkts_buf_size / buffer_threshold),
+		      SMALL_PKTS_BUF_COUNT_MIN, SMALL_PKTS_BUF_COUNT_MAX));
+
+	dev->ringparam.tx_pending = clamp_t(unsigned int, ring->tx_pending,
+					    1, CXI_MAX_CQ_COUNT);
+
+	if (netif_running(ndev)) {
+		hw_cleanup(dev);
+		rc = hw_setup(dev);
+		if (rc)
+			netdev_err(ndev,
+				   "Failed to restart after ringparam change\n");
+	}
+
+	return rc;
+}
+
+static int cxi_set_phys_id(struct net_device *ndev,
+			   enum ethtool_phys_id_state state)
+{
+	struct cxi_eth *dev = netdev_priv(ndev);
+
+	switch (state) {
+	case ETHTOOL_ID_ACTIVE:
+		cxi_set_led_beacon(dev->cxi_dev, true);
+		return 0;
+
+	case ETHTOOL_ID_INACTIVE:
+		cxi_set_led_beacon(dev->cxi_dev, false);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+const struct ethtool_ops cxi_eth_ethtool_ops = {
+	.supported_ring_params =
+#ifdef ETHTOOL_RING_USE_CQE_SIZE
+	ETHTOOL_RING_USE_CQE_SIZE |
+#endif
+	ETHTOOL_RING_USE_RX_BUF_LEN,
+	.get_drvinfo	= cxi_get_drvinfo,
+	.get_sset_count	= cxi_get_sset_count,
+	.get_strings	= cxi_get_strings,
+	.get_rxnfc	= cxi_get_rxnfc,
+	.get_channels	= cxi_get_channels,
+	.set_channels	= cxi_set_channels,
+	.get_rxfh_indir_size = cxi_get_rxfh_indir_size,
+	.get_rxfh       = cxi_get_rxfh,
+	.set_rxfh       = cxi_set_rxfh,
+	.get_rxfh_key_size = cxi_get_rxfh_key_size,
+	.get_module_info = cxi_get_module_info,
+	.get_module_eeprom = cxi_get_module_eeprom,
+	.get_module_eeprom_by_page = cxi_get_module_eeprom_by_page,
+	.flash_device   = cxi_flash_device,
+	.get_link	= ethtool_op_get_link,
+	.get_link_ksettings = cxi_get_link_ksettings,
+	.set_link_ksettings = cxi_set_link_ksettings,
+	.get_priv_flags     = cxi_get_priv_flags,
+	.set_priv_flags     = cxi_set_priv_flags,
+	.get_ethtool_stats  = cxi_get_ethtool_stats,
+	.get_ts_info        = cxi_get_ts_info,
+	.set_phys_id        = cxi_set_phys_id,
+	.get_pauseparam     = cxi_get_pauseparam,
+	.set_pauseparam     = cxi_set_pauseparam,
+	.get_fecparam       = cxi_get_fecparam,
+	.set_fecparam       = cxi_set_fecparam,
+	.get_ringparam      = cxi_get_ringparam,
+	.set_ringparam      = cxi_set_ringparam,
+};