Re: [RFC PATCH 1/2] mfd: Add SEL PCI Virtual Multifunction (PVMF) support

On Mon, 28 Oct 2024, Robert Joslyn wrote:

No changes since 2017?

Why upstream this now?

+ * 2350 NE Hopkins Court, Pullman, WA 99163 USA
+ * SEL Open Source [off-list ref]
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/property.h>
+#include <linux/mfd/core.h>
+
+#include "selpvmf-cvp.h"
+
+#define PCI_VENDOR_ID_SEL	0x1aa9
+#define PCI_DEVICE_ID_B1190	0x0014 /* SEL-3390T */
+#define PCI_DEVICE_ID_B2093	0x0015 /* SEL-3350 mainboard */
+#define PCI_DEVICE_ID_B2077	0x0018 /* SEL-3390E4 */
+#define PCI_DEVICE_ID_B2091	0x0019 /* SEL-2241-2 */
+
+#define B1190_NUM_ETHERNET 2
+#define B2077_NUM_ETHERNET 4
+#define B2091_NUM_ETHERNET 5
+#define B2093_NUM_ETHERNET 5
+#define SELETH_NUM_RESOURCES 4
+
+#define SELETH_LEN	0x1000
+
+static bool skipcvp;
+module_param(skipcvp, bool, 0644);
+MODULE_PARM_DESC(skipcvp, "Skip firmware load via CvP.");
+
+/**
+ * sel_create_cell() - Create an MFD cell and add it to the device.
+ * @pdev: The PCI device to operate on
+ * @name: MFD cell name
+ * @start: Start address of memory resource
+ * @length: Length of memory resource
+ * @msix_start: MSIX vector number to start from.
+ * @resources: Pointer to resources to add to the cell.
+ * @num_resources: Number of resources. The first resource is assumed to
+ *                 be memory, all other resources are IRQs.
+ */
+static int sel_create_cell(struct pci_dev *pdev,

This is misleading.  It should be sel_register_device.

+			   const char *name,

This never changes.  Why supply it as a variable?

+			   unsigned int start,
+			   unsigned int length,

This never changes.  Why supply it as a variable?

+			   unsigned int msix_start,

This never changes.  Why supply it as a variable?

+			   struct resource *resources,
+			   unsigned int num_resources)

This never changes.  Why supply it as a variable?

+{
+	struct mfd_cell cell;
+	unsigned int msix;
+	int rc;
+	int i;
+
+	if (!resources || num_resources < 1)

How would this be possible?

+		return -EINVAL;
+
+	/* If MSI-X is enabled, assume we're using it and start at the

Malformed multi-line comment.

+	 * provided vector number and iterate through the interrupts
+	 * when adding IRQ resources. Otherwise leave the interrupt
+	 * index at zero for legacy IRQs.
+	 */
+	if (pdev->msix_enabled)
+		msix = msix_start;
+	else
+		msix = 0;

Preinitialise then drop the else.

+	/* First resource is always a memory resource */
+	resources[0].start = start;
+	resources[0].end = start + length - 1;
+	resources[0].flags = IORESOURCE_MEM;
+
+	/* Any additional resources are IRQs */
+	for (i = 1; i < num_resources; i++) {
+		resources[i].start = pci_irq_vector(pdev, msix);
+		resources[i].end = pci_irq_vector(pdev, msix);

Does pci_irq_vector(pdev, msix) change between calls?

+		resources[i].flags = IORESOURCE_IRQ;
+		if (pdev->msix_enabled)
+			msix++;
+	}
+
+	cell.name = name;
+	cell.num_resources = num_resources;
+	cell.resources = resources;
+
+	rc = mfd_add_devices(&pdev->dev, PLATFORM_DEVID_AUTO, &cell,
+			     1, NULL, 0, NULL);
+
+	return rc;
+}
+
+/**
+ * create_eth_cell - Create and add an MFD cell for an selnet component
+ *
+ * @pdev: PCI device
+ * @offset: Offset from BAR 0 of the component
+ * @msix_num: Starting MSI-X vector number
+ */
+static int create_eth_cell(struct pci_dev *pdev,
+			   unsigned int offset,
+			   unsigned int msix_num)
+{
+	struct resource resources[SELETH_NUM_RESOURCES] = {0};
+
+	return sel_create_cell(pdev,

This is only called from here, thus bring all of the sel_create_cell()
stuff into here instead.

+			       "selpcimac",
+			       pdev->resource[0].start + offset,
+			       SELETH_LEN,
+			       msix_num,
+			       resources,
+			       SELETH_NUM_RESOURCES);
+}
+
+/**
+ * b1190_alloc_mfd_cells - Allocate and initialize MFD cell descriptions
+ *
+ * @pdev: PCI device for the b1190
+ *
+ * +----------------------------------------+
+ * |          | Resources |     Address     |
+ * |  Device  | Mem | IRQ | Start  | Length |
+ * |----------+-----+-----+--------+--------|
+ * | Ethernet |  1  |  3  | 0x0000 | 0x1000 |
+ * | Ethernet |  1  |  3  | 0x1000 | 0x1000 |
+ * | seltime  |  1  |  1  | 0x3000 | 0x1000 |
+ * | upgrades |  1  |  0  | 0x4000 | 0x4000 |
+ * +----------------------------------------+
+ */
+static int b1190_alloc_mfd_cells(struct pci_dev *pdev)
+{
+	unsigned int offset;
+	int rc;

'ret' is more common.

+	int i;

Use for (int i = 0 ... instead.

+	/* Ethernet */
+	offset = 0x0000;

Initialise this during declaration.

+	for (i = 0; i < B1190_NUM_ETHERNET; i++) {
+		rc = create_eth_cell(pdev, offset, i * 3);
+		if (rc)
+			goto out_fail;
+		offset += SELETH_LEN;
+	}
+
+out_fail:
+	return rc;
+}

These functions are always identical.  Create a single function and pass
the number of ports into it and remove the rest.

+
+/**
+ * b2077_alloc_mfd_cells - Allocate and initialize MFD cell descriptions
+ *
+ * @pdev: PCI device for the b2077
+ *
+ * +----------------------------------------+
+ * |          | Resources |     Address     |
+ * |  Device  | Mem | IRQ | Start  | Length |
+ * |----------+-----+-----+--------+--------|
+ * | Ethernet |  1  |  3  | 0x0000 | 0x1000 |
+ * | Ethernet |  1  |  3  | 0x1000 | 0x1000 |
+ * | Ethernet |  1  |  3  | 0x2000 | 0x1000 |
+ * | Ethernet |  1  |  3  | 0x3000 | 0x1000 |
+ * | upgrades |  1  |  0  | 0x8000 | 0x4000 |
+ * | seltime  |  1  |  1  | 0xc000 | 0x1000 |
+ * +----------------------------------------+
+ */
+static int b2077_alloc_mfd_cells(struct pci_dev *pdev)
+{
+	unsigned int offset;
+	int rc;
+	int i;
+
+	/* Ethernet */
+	offset = 0x0000;
+	for (i = 0; i < B2077_NUM_ETHERNET; i++) {
+		rc = create_eth_cell(pdev, offset, i * 3);
+		if (rc)
+			goto out_fail;
+		offset += SELETH_LEN;
+	}
+
+out_fail:
+	return rc;
+}
+
+/*
+ * b2091_alloc_mfd_cells - Allocate and initialize MFD cell descriptions
+ *
+ * @pdev: PCI device for the b2091
+ *
+ * +------------------------------------------+
+ * |            | Resources |     Address     |
+ * |   Device   | Mem | IRQ | Start  | Length |
+ * |------------+-----+-----+--------+--------|
+ * | Ethernet   |  1  |  3  | 0x0000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x1000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x2000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x3000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x4000 | 0x1000 |
+ * | upgrades   |  1  |  0  | 0x8000 | 0x4000 |
+ * | seltime    |  1  |  1  | 0xc000 | 0x1000 |
+ * | multiuart  |  1  |  1  | 0xf000 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf100 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf200 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf300 | 0x0100 |
+ * +------------------------------------------+
+ */
+static int b2091_alloc_mfd_cells(struct pci_dev *pdev)
+{
+	unsigned int offset;
+	int rc;
+	int i;
+
+	/* Ethernet */
+	offset = 0x0000;
+	for (i = 0; i < B2091_NUM_ETHERNET; i++) {
+		rc = create_eth_cell(pdev, offset, i * 3);
+		if (rc)
+			goto out_fail;
+		offset += SELETH_LEN;
+	}
+
+out_fail:
+	return rc;
+}
+
+/**
+ * b2093_alloc_mfd_cells - Allocate and initialize MFD cell descriptions
+ *
+ * @pdev: PCI device for the b2093
+ *
+ * +------------------------------------------+
+ * |            | Resources |     Address     |
+ * |   Device   | Mem | IRQ | Start  | Length |
+ * |------------+-----+-----+--------+--------|
+ * | Ethernet   |  1  |  3  | 0x0000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x1000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x2000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x3000 | 0x1000 |
+ * | Ethernet   |  1  |  3  | 0x4000 | 0x1000 |
+ * | upgrades   |  1  |  0  | 0x8000 | 0x4000 |
+ * | seltime    |  1  |  1  | 0xc000 | 0x1000 |
+ * | contact IO |  1  |  0  | 0xe000 | 0x1000 |
+ * | multiuart  |  1  |  1  | 0xf000 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf100 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf200 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf300 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf400 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf500 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf600 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf700 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf800 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xf900 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xfa00 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xfb00 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xfc00 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xfd00 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xfe00 | 0x0100 |
+ * | multiuart  |  1  |  1  | 0xff00 | 0x0100 |
+ * +------------------------------------------+
+ */
+static int b2093_alloc_mfd_cells(struct pci_dev *pdev)
+{
+	unsigned int offset;
+	int rc;
+	int i;
+
+	/* Ethernet */
+	offset = 0x0000;
+	for (i = 0; i < B2093_NUM_ETHERNET; i++) {
+		rc = create_eth_cell(pdev, offset, i * 3);
+		if (rc)
+			goto out_fail;
+		offset += SELETH_LEN;
+	}
+
+out_fail:
+	return rc;
+}
+
+/*
+ * selpvmf_probe() - Probe function for the device.

None of these headers are really necessary.  This is not an external API.

+ *
+ * @pdev: PCI device being probed.
+ * @ent: Entry in the pci_tbl for the device.
+ *
+ * Return: 0 for success, otherwise the appropriate negative error code
+ */
+static int selpvmf_probe(struct pci_dev *pdev, struct pci_device_id const *ent)
+{
+	typedef int (*alloc_fn_t)(struct pci_dev *pdev);
+	alloc_fn_t alloc_fn = (alloc_fn_t)ent->driver_data;
+	u16 cvp_capable;
+	int num_msix;
+	int rc = 0;
+
+	rc = pcim_enable_device(pdev);

Could it be that the device isn't ready?

No use for -EPROBE_DEFER?

+	if (rc != 0)
+		return rc;
+
+	/* Perform CvP program if necessary */
+	cvp_capable = pci_find_ext_capability(pdev, CVP_CAPABILITY_ID);
+	if (!skipcvp && cvp_capable) {

The '_' inconsistency is bothering me.

As is the placement of "cvp".

+		rc = cvp_start(pdev);
+		if (rc != 0) {
+			dev_err(&pdev->dev, "Error: CvP failed: 0x%x\n", rc);

This is a user facing error message and I still don't know what CvP is.

Please be more forthcoming.

+			goto out_disable;
+		}
+		dev_info(&pdev->dev, "CvP Successful!\n");

This serves no purpose.  Please keep the log as clean as possible.

+	}
+
+	pci_set_master(pdev);
+
+	/* Only MSI-X and legacy interrupts are supported. Try to get

Malformed multi-line comment.

+	 * MSI-X, if we don't get them all, fall back to legacy.

Full stop after MSI-X.

+	 */
+	num_msix = pci_msix_vec_count(pdev);

'\n' here.

+	rc = pci_alloc_irq_vectors(pdev, num_msix, num_msix, PCI_IRQ_MSIX);
+	if (rc != num_msix) {
+		dev_warn(&pdev->dev,
+			 "Failed to get MSI-X, falling back to legacy\n");

Who is going to care about that?

'\n' here.

+		rc = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_INTX);
+		if (rc != 1) {
+			rc = -ENOSPC;

I think you should be returning rc as-is here.

+			goto out_disable;
+		}
+	}
+
+	rc = alloc_fn(pdev);

These call-backs are horrible.  Please don't do this.

+	if (rc < 0) {
+		dev_err(&pdev->dev, "ERROR: Failed to add child cells\n");

"Failed to register child devices"

And it should come after the call to mfd_add_devices() not here.

+		goto out_free_irq;
+	}
+
+	dev_info(&pdev->dev, "Successfully added new devices\n");

Remove this please.

+	return 0;
+
+out_free_irq:
+	pci_free_irq_vectors(pdev);
+out_disable:
+	pci_clear_master(pdev);
+	return rc;
+}
+
+/*
+ * selpvmf_remove - Remove a selpvmf enabled PCI device from the system
+ *
+ * @pdev: PCI device to be removed
+ */
+static void selpvmf_remove(struct pci_dev *pdev)
+{
+	mfd_remove_devices(&pdev->dev);

Use devm_* instead,

+	pci_free_irq_vectors(pdev);
+	pci_clear_master(pdev);
+}
+
+/*
+ * selpvmf_shutdown() - Tears down a selpvmf card.
+ *
+ * @pdev: Pointer to the device structure describing the port.
+ */
+static void selpvmf_shutdown(struct pci_dev *pdev)
+{
+	mfd_remove_devices(&pdev->dev);

As above.

+}
+
+/*
+ * selpvmf_suspend - Suspend the device in preparation for entering a
+ * a low power state.
+ *
+ * @dev: Pointer to the device structure describing the port.
+ *
+ * Return: This function always indicates success.
+ *
+ * Most of the work should be handled by the child devices of this
+ * driver. There is no specific action necessary by the card, so when
+ * the child devices are done suspending, just do a generic PCI saving
+ * of state.
+ */
+static int selpvmf_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+
+	pci_save_state(pdev);
+
+	return 0;
+}
+
+/*
+ * selpvmf_resume() - Resume a selpvmf device that has previously been
+ * suspended
+ *
+ * @dev: Pointer to the device structure describing the port.
+ *
+ * Return: This function always indicates success.
+ */
+static int selpvmf_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	u16 cvp_capable;
+	int rc;
+
+	/* Turn the card on by requesting a transition to D0 before
+	 * restoring the PCI device state and enabling the UARTs
+	 */
+	pci_set_power_state(pdev, PCI_D0);
+
+	/* Restore so we can do IOMEM based CvP */
+	pci_restore_state(pdev);
+
+	rc = pcim_enable_device(pdev);
+	if (rc != 0) {
+		/* We are not really supposed to return an error from this
+		 * function, so if the enable fails, just print some info
+		 * to aid in troubleshooting.
+		 */

What?  It returns an int for a reason, please use it.

+		dev_err(&pdev->dev,
+			"Unable to enable the device. Continuing anyway\n");
+	}
+
+	/* Perform CvP program if necessary */
+	cvp_capable = pci_find_ext_capability(pdev, CVP_CAPABILITY_ID);
+	if (!skipcvp && cvp_capable) {
+		pci_save_state(pdev);
+
+		rc = cvp_start(pdev);
+		if (rc != 0) {
+			dev_err(&pdev->dev,
+				"Error: CvP failed (resume): 0x%x\n", rc);

You can use up to 100-chars to prevent this kind of wrapping.

+		}
+		dev_info(&pdev->dev, "CvP done (resume): 0x%x\n", rc);

Remove this please.

+
+		pci_restore_state(pdev);
+	}
+
+	pci_set_master(pdev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops selpvmf_pm_ops = {
+	SYSTEM_SLEEP_PM_OPS(selpvmf_suspend, selpvmf_resume)
+};
+
+static struct pci_device_id selpvmf_pci_tbl[] = {
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_SEL, PCI_DEVICE_ID_B1190),
+		.driver_data = (kernel_ulong_t)b1190_alloc_mfd_cells,

Pass an identifier here instead of call-back functions.

+	},
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_SEL, PCI_DEVICE_ID_B2077),
+		.driver_data = (kernel_ulong_t)b2077_alloc_mfd_cells,
+	},
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_SEL, PCI_DEVICE_ID_B2091),
+		.driver_data = (kernel_ulong_t)b2091_alloc_mfd_cells,
+	},
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_SEL, PCI_DEVICE_ID_B2093),
+		.driver_data = (kernel_ulong_t)b2093_alloc_mfd_cells,
+	},
+	{0,}

What is this?  Where have you seen that before?

I'm not sure what all of this is, but I suggest that it doesn't live
here.  drivers/firmware or drivers/fpga perhaps?

Do you really need the whole postal address here?

I suggest this is legacy of a time well past.

+ *
+ * Support for Configuration via Protocol (CvP) on SEL devices using
+ * Altera FPGAs.
+ */
+
+#include <linux/firmware.h>
+#include <linux/pci.h>
+#include <linux/sched.h>
+
+#include "selpvmf-cvp.h"
+
+#define MIN_WAIT 2 /* number of jiffies for unit wait period */

Comments should start with an upper case char.

+#define US_PER_JIFFY  (1000000 / HZ) /* number of microseconds per jiffy */

Please tab all of these out.

+#define CVP_WAIT_TIMEOUT 1000000 /* Wait timeout in microseconds */
+#define BYTES_PER_DWORD	4
+#define CORE_FIRMWARE_FILE_BASEDIR "sel/"
+#define CORE_FIRMWARE_FILE_EXT ".cvp"
+
+#define CVP_STATUS_OFFSET	0x0000001C
+#define CVP_CONFIG_READY	BIT(18)
+#define CVP_CONFIG_ERROR	BIT(19)
+#define CVP_EN			BIT(20)
+#define USERMODE		BIT(21)
+#define PLD_CLK_IN_USE		BIT(24)
+
+#define CVP_MODE_CONTROL_OFFSET		0x00000020
+#define CVP_MODE			BIT(0)
+#define HIP_CLK_SEL			BIT(1)
+#define CVP_NUMCLKS_MASK		GENMASK(15, 8)
+#define CVP_NUMCLKS_UNCMPRSSD_UNENCRYPT	BIT(8)
+#define CVP_NUMCLKS_COMPRESSED		BIT(11)
+#define CVP_NUMCLKS_ENCRYPTED		BIT(10)
+
+#define CVP_DATA_OFFSET	0x00000028
+
+#define CVP_PROGRAMMING_CTRL_OFFSET	0x0000002C
+#define CVP_CONFIG			BIT(0)
+#define START_XFER			BIT(1)
+
+#define CVP_UNCORRECTABLE_IE_STATUS_OFFSET	0x00000034
+#define CVP_CONFIG_ERROR_LATCHED		BIT(5)

I'm going to stop here.  Please address the issues before resubmitting.

-- 
Lee Jones [李琼斯]