[PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

Subsystems: pci driver for synopsys designware, pci native host bridge and endpoint drivers, pci subsystem, the rest

STALE2570d

27 messages, 4 authors, 2019-08-27 · open the first message on its own page

[PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:47:50

Add multiple PFs support for DWC, different PF have different config space,
we use pf-offset property which get from the DTS to access the different pF
config space.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97 +++++++++++++---------
 drivers/pci/controller/dwc/pcie-designware.c    | 105 ++++++++++++++++++++++--
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
 	pci_epc_linkup(epc);
 }
 
-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar,
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no,
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;
 
-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0);
@@ -37,7 +39,12 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar,
 
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 {
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }
 
 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,
@@ -78,28 +85,29 @@ static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 
 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID, hdr->vendorid);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID, hdr->deviceid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID, hdr->revid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG, hdr->progif_code);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_VENDOR_ID,
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID, hdr->subsys_id);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);
 
 	return 0;
 }
 
-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)
 {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
 		return -EINVAL;
 	}
 
-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar, cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n");
@@ -125,7 +133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
 	return 0;
 }
 
-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)
 {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
 		return -EINVAL;
 	}
 
-	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win, PCIE_ATU_TYPE_MEM,
+				     phys_addr, pci_addr, size);
 
 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr;
@@ -154,7 +163,7 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];
 
-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);
 
 	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
 	clear_bit(atu_index, ep->ib_window_map);
@@ -170,14 +179,16 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;
 
 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;
 
-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type);
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
 
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no,
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 
-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msi_cap)
 		return -EINVAL;
 
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msi_cap)
 		return -EINVAL;
 
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK;
@@ -287,12 +300,13 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msix_cap)
 		return -EINVAL;
 
-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msix_cap)
 		return -EINVAL;
 
-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 		return -EINVAL;
 
 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
@@ -439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 }
 
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc;
@@ -447,16 +463,17 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;
 
-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
 
-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
@@ -592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);
 
-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions", &epc->max_functions);
 	if (ret < 0)
 		epc->max_functions = 1;
 
+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 	dw_pcie_writel_atu(pci, offset + reg, val);
 }
 
+static void dw_pcie_prog_ep_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no,
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_BASE,
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_BASE,
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
+				 type | PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
+}
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size)
@@ -194,6 +231,51 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 }
 
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no, index, type,
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
+}
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 			       u64 cpu_addr, u64 pci_addr, u32 size)
 {
@@ -252,8 +334,8 @@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 	dw_pcie_writel_atu(pci, offset + reg, val);
 }
 
-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8 func_no,
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)
 {
 	int type;
@@ -275,8 +357,10 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
 		return -EINVAL;
 	}
 
-	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type);
+	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type |
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
 
@@ -297,14 +381,15 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
 	return -EBUSY;
 }
 
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type)
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;
 
 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar,
 						       cpu_addr, as_type);
 
 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_INBOUND |
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
 		return -EINVAL;
 	}
 
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
 
 	/*
 	 * Make sure ATU enable takes effect before any subsequent config
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci);
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type);
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);
 void dw_pcie_setup(struct dw_pcie *pci);
diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h
index f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP controller */
 	spinlock_t			lock;
-- 
2.9.5

[PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:47:49

Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h    | 14 ++++++++++++++
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 	return 0;
 }
 
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no,
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
 {
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 2b291e8..cd903e9 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -88,6 +88,11 @@
 #define PCIE_MISC_CONTROL_1_OFF		0x8BC
 #define PCIE_DBI_RO_WR_EN		BIT(0)
 
+#define PCIE_MSIX_DOORBELL		0x948
+#define PCIE_MSIX_DOORBELL_PF_SHIFT	24
+#define PCIE_MSIX_DOORBELL_VF_SHIFT	16
+#define PCIE_MSIX_DOORBELL_VF_ACTIVE	BIT(15)
+
 /*
  * iATU Unroll-specific register definitions
  * From 4.80 core version the address translation will be made by unroll
@@ -399,6 +404,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u8 interrupt_num);
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u16 interrupt_num);
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no,
+				       u16 interrupt_num);
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar);
 #else
 static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
@@ -431,6 +438,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 	return 0;
 }
 
+static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep,
+						     u8 func_no,
+						     u16 interrupt_num)
+{
+	return 0;
+}
+
 static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 {
 }
-- 
2.9.5

[PATCH 04/10] dt-bindings: pci: layerscape-pci: add compatible strings for ls1088a and ls2088a

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:47:53

Add compatible strings for ls1088a and ls2088a.

Signed-off-by: Xiaowei Bao <redacted>
---
 Documentation/devicetree/bindings/pci/layerscape-pci.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt
index e20ceaa..16f592e 100644
--- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt
+++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt
@@ -22,7 +22,10 @@ Required properties:
         "fsl,ls1043a-pcie"
         "fsl,ls1012a-pcie"
   EP mode:
-	"fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep"
+	"fsl,ls-pcie-ep"
+	"fsl,ls1046a-pcie-ep"
+	"fsl,ls1088a-pcie-ep"
+	"fsl,ls2088a-pcie-ep"
 - reg: base addresses and lengths of the PCIe controller register blocks.
 - interrupts: A list of interrupt outputs of the controller. Must contain an
   entry for each entry in the interrupt-names property.
-- 
2.9.5

[PATCH 05/10] PCI: layerscape: Modify the way of getting capability with different PEX

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:47:55

The different PCIe controller in one board may be have different
capability of MSI or MSIX, so change the way of getting the MSI
capability, make it more flexible.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 28 +++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index be61d96..9404ca0 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -22,6 +22,7 @@
 
 struct ls_pcie_ep {
 	struct dw_pcie		*pci;
+	struct pci_epc_features	*ls_epc;
 };
 
 #define to_ls_pcie_ep(x)	dev_get_drvdata((x)->dev)
@@ -40,25 +41,26 @@ static const struct of_device_id ls_pcie_ep_of_match[] = {
 	{ },
 };
 
-static const struct pci_epc_features ls_pcie_epc_features = {
-	.linkup_notifier = false,
-	.msi_capable = true,
-	.msix_capable = false,
-};
-
 static const struct pci_epc_features*
 ls_pcie_ep_get_features(struct dw_pcie_ep *ep)
 {
-	return &ls_pcie_epc_features;
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
+
+	return pcie->ls_epc;
 }
 
 static void ls_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
 	enum pci_barno bar;
 
 	for (bar = BAR_0; bar <= BAR_5; bar++)
 		dw_pcie_ep_reset_bar(pci, bar);
+
+	pcie->ls_epc->msi_capable = ep->msi_cap ? true : false;
+	pcie->ls_epc->msix_capable = ep->msix_cap ? true : false;
 }
 
 static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
@@ -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct dw_pcie *pci;
 	struct ls_pcie_ep *pcie;
+	struct pci_epc_features *ls_epc;
 	struct resource *dbi_base;
 	int ret;
 
@@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	if (!pci)
 		return -ENOMEM;
 
+	ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL);
+	if (!ls_epc)
+		return -ENOMEM;
+
 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
 	if (IS_ERR(pci->dbi_base))
@@ -139,6 +146,13 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	pci->ops = &ls_pcie_ep_ops;
 	pcie->pci = pci;
 
+	ls_epc->linkup_notifier = false,
+	ls_epc->msi_capable = true,
+	ls_epc->msix_capable = true,
+	ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4),
+
+	pcie->ls_epc = ls_epc;
+
 	platform_set_drvdata(pdev, pcie);
 
 	ret = ls_add_pcie_ep(pcie, pdev);
-- 
2.9.5

[PATCH 07/10] PCI: layerscape: Fix some format issue of the code

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:47:58

Fix some format issue of the code in EP driver.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index a0cd5ff..2ada445 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -64,7 +64,7 @@ static void ls_pcie_ep_init(struct dw_pcie_ep *ep)
 }
 
 static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
-				  enum pci_epc_irq_type type, u16 interrupt_num)
+				enum pci_epc_irq_type type, u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 
@@ -89,7 +89,7 @@ static const struct dw_pcie_ep_ops pcie_ep_ops = {
 };
 
 static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie,
-					struct platform_device *pdev)
+				 struct platform_device *pdev)
 {
 	struct dw_pcie *pci = pcie->pci;
 	struct device *dev = pci->dev;
-- 
2.9.5

[PATCH 08/10] dt-bindings: PCI: Add the pf-offset property

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:48:00

Add the pf-offset property for multiple PF.

Signed-off-by: Xiaowei Bao <redacted>
---
 Documentation/devicetree/bindings/pci/designware-pcie.txt | 1 +
 1 file changed, 1 insertion(+)
diff --git a/Documentation/devicetree/bindings/pci/designware-pcie.txt b/Documentation/devicetree/bindings/pci/designware-pcie.txt
index 5561a1c..d658687 100644
--- a/Documentation/devicetree/bindings/pci/designware-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/designware-pcie.txt
@@ -43,6 +43,7 @@ RC mode:
 
 EP mode:
 - max-functions: maximum number of functions that can be configured
+- pf-offset: the offset of each PF's config space
 
 Example configuration:
 
-- 
2.9.5

[PATCH 09/10] arm64: dts: layerscape: Add PCIe EP node for ls1088a

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:48:03

Add PCIe EP node for ls1088a to support EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi | 32 ++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
index dfbead4..434a76c 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
@@ -471,6 +471,18 @@
 			status = "disabled";
 		};
 
+		pcie_ep@3400000 {
+			compatible = "fsl,ls1088a-pcie-ep","fsl,ls-pcie-ep";
+			reg = <0x00 0x03400000 0x0 0x00100000
+			       0x20 0x00000000 0x8 0x00000000>;
+			reg-names = "regs", "addr_space";
+			num-ib-windows = <24>;
+			num-ob-windows = <128>;
+			max-functions = /bits/ 8 <2>;
+			pf-offset = <0x20000>;
+			status = "disabled";
+		};
+
 		pcie@3500000 {
 			compatible = "fsl,ls1088a-pcie";
 			reg = <0x00 0x03500000 0x0 0x00100000   /* controller registers */
@@ -497,6 +509,16 @@
 			status = "disabled";
 		};
 
+		pcie_ep@3500000 {
+			compatible = "fsl,ls1088a-pcie-ep","fsl,ls-pcie-ep";
+			reg = <0x00 0x03500000 0x0 0x00100000
+			       0x28 0x00000000 0x8 0x00000000>;
+			reg-names = "regs", "addr_space";
+			num-ib-windows = <6>;
+			num-ob-windows = <8>;
+			status = "disabled";
+		};
+
 		pcie@3600000 {
 			compatible = "fsl,ls1088a-pcie";
 			reg = <0x00 0x03600000 0x0 0x00100000   /* controller registers */
@@ -523,6 +545,16 @@
 			status = "disabled";
 		};
 
+		pcie_ep@3600000 {
+			compatible = "fsl,ls1088a-pcie-ep","fsl,ls-pcie-ep";
+			reg = <0x00 0x03600000 0x0 0x00100000
+			       0x30 0x00000000 0x8 0x00000000>;
+			reg-names = "regs", "addr_space";
+			num-ib-windows = <6>;
+			num-ob-windows = <8>;
+			status = "disabled";
+		};
+
 		smmu: iommu@5000000 {
 			compatible = "arm,mmu-500";
 			reg = <0 0x5000000 0 0x800000>;
-- 
2.9.5

[PATCH 10/10] misc: pci_endpoint_test: Add LS1088a in pci_device_id table

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:48:06

Add LS1088a in pci_device_id table so that pci-epf-test can be used
for testing PCIe EP in LS1088a.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/misc/pci_endpoint_test.c | 1 +
 1 file changed, 1 insertion(+)
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index 6e208a0..d531951 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -793,6 +793,7 @@ static const struct pci_device_id pci_endpoint_test_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x80c0) },
 	{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
 	  .driver_data = (kernel_ulong_t)&am654_data
-- 
2.9.5

[PATCH 06/10] PCI: layerscape: Modify the MSIX to the doorbell way

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:48:12

The layerscape platform use the doorbell way to trigger MSIX
interrupt in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index 9404ca0..a0cd5ff 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -74,7 +74,8 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
 	case PCI_EPC_IRQ_MSI:
 		return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num);
 	case PCI_EPC_IRQ_MSIX:
-		return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num);
+		return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no,
+							  interrupt_num);
 	default:
 		dev_err(pci->dev, "UNKNOWN IRQ type\n");
 		return -EINVAL;
-- 
2.9.5

[PATCH 03/10] PCI: designware-ep: Move the function of getting MSI capability forward

From: Xiaowei Bao <hidden>
Date: 2019-08-15 08:48:19

Move the function of getting MSI capability to the front of init
function, because the init function of the EP platform driver will use
the return value by the function of getting MSI capability.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index e3a7cdf..0c27c7b 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -631,6 +631,10 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	if (ret < 0)
 		epc->pf_offset = 0;
 
+	ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI);
+
+	ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX);
+
 	if (ep->ops->ep_init)
 		ep->ops->ep_init(ep);
 
@@ -647,9 +651,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 		dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n");
 		return -ENOMEM;
 	}
-	ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI);
-
-	ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX);
 
 	offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
 	if (offset) {
-- 
2.9.5

Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Andrew Murray <hidden>
Date: 2019-08-15 11:31:35

On Thu, Aug 15, 2019 at 04:37:07PM +0800, Xiaowei Bao wrote:
Add multiple PFs support for DWC, different PF have different config space,
we use pf-offset property which get from the DTS to access the different pF
config space.
Thanks for the patch. I haven't seen a cover letter for this series, is there
one missing?

quoted hunk
Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97 +++++++++++++---------
 drivers/pci/controller/dwc/pcie-designware.c    | 105 ++++++++++++++++++++++--
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
 	pci_epc_linkup(epc);
 }
 
-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar,
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no,
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;
 
-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
I think I'd rather see this arithmetic (and the one for determining pf_base)
inside a macro or inline header function. This would make this code more readable
and reduce the chances of an error by avoiding duplication of code.

For example look at cdns_pcie_ep_fn_writeb and ROCKCHIP_PCIE_EP_FUNC_BASE for
examples of other EP drivers that do this.

quoted hunk
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0);
@@ -37,7 +39,12 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar,
 
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 {
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }
 
 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,
@@ -78,28 +85,29 @@ static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 
 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID, hdr->vendorid);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID, hdr->deviceid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID, hdr->revid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG, hdr->progif_code);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_VENDOR_ID,
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID, hdr->subsys_id);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);
 
 	return 0;
 }
 
-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)
 {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
 		return -EINVAL;
 	}
 
-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar, cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n");
@@ -125,7 +133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar,
 	return 0;
 }
 
-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)
 {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
 		return -EINVAL;
 	}
 
-	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win, PCIE_ATU_TYPE_MEM,
+				     phys_addr, pci_addr, size);
 
 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr;
@@ -154,7 +163,7 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];
 
-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);
 
 	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
 	clear_bit(atu_index, ep->ib_window_map);
@@ -170,14 +179,16 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;
 
 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;
 
-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type);
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
 
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no,
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 
-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msi_cap)
 		return -EINVAL;
 
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msi_cap)
 		return -EINVAL;
 
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK;
@@ -287,12 +300,13 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msix_cap)
 		return -EINVAL;
 
-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts)
 {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;
 
 	if (!ep->msix_cap)
 		return -EINVAL;
 
-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 		return -EINVAL;
 
 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
@@ -439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 }
 
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc;
@@ -447,16 +463,17 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;
 
-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;
 
-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
@@ -592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);
 
-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions", &epc->max_functions);
 	if (ret < 0)
 		epc->max_functions = 1;
 
+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
Bad things will likely happen if max_functions > 1 and pf-offset isn't set.
I think the driver should bail in this situation. It would be very easy
for someone to misconfigure this.

quoted hunk
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 	dw_pcie_writel_atu(pci, offset + reg, val);
 }
 
+static void dw_pcie_prog_ep_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no,
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_BASE,
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_BASE,
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
+				 type | PCIE_ATU_FUNC_NUM(func_no));
With the exception of this line, the rest of this function is identical to
dw_pcie_prog_outbound_atu_unroll.
quoted hunk
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
+}
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size)
@@ -194,6 +231,51 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
 }
 
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no, index, type,
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
The same here, this is identical to dw_pcie_prog_outbound_atu with the
exception of this line.

Is there a way you can avoid all of this duplicated code?

Thanks,

Andrew Murray 
quoted hunk
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
+}
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 			       u64 cpu_addr, u64 pci_addr, u32 size)
 {
@@ -252,8 +334,8 @@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg,
 	dw_pcie_writel_atu(pci, offset + reg, val);
 }
 
-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8 func_no,
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)
 {
 	int type;
@@ -275,8 +357,10 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
 		return -EINVAL;
 	}
 
-	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type);
+	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type |
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
 
@@ -297,14 +381,15 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
 	return -EBUSY;
 }
 
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type)
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;
 
 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar,
 						       cpu_addr, as_type);
 
 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_INBOUND |
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
 		return -EINVAL;
 	}
 
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
 
 	/*
 	 * Make sure ATU enable takes effect before any subsequent config
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci);
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type);
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);
 void dw_pcie_setup(struct dw_pcie *pci);
diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h
index f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP controller */
 	spinlock_t			lock;
-- 
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Andrew Murray <hidden>
Date: 2019-08-15 11:53:45

On Thu, Aug 15, 2019 at 04:37:08PM +0800, Xiaowei Bao wrote:
quoted hunk
Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h    | 14 ++++++++++++++
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 	return 0;
 }
 
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no,
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
Have I understood correctly that the hardware provides an alternative mechanism
that allows for raising MSI-X interrupts without the bother of reading the
capabilities registers?

If so is there any good reason to keep dw_pcie_ep_raise_msix_irq? (And thus use
it in dw_plat_pcie_ep_raise_irq also)?

quoted hunk
 {
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 2b291e8..cd903e9 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -88,6 +88,11 @@
 #define PCIE_MISC_CONTROL_1_OFF		0x8BC
 #define PCIE_DBI_RO_WR_EN		BIT(0)
 
+#define PCIE_MSIX_DOORBELL		0x948
+#define PCIE_MSIX_DOORBELL_PF_SHIFT	24
+#define PCIE_MSIX_DOORBELL_VF_SHIFT	16
+#define PCIE_MSIX_DOORBELL_VF_ACTIVE	BIT(15)
The _VF defines are not used, I'd suggest removing them.

Thanks,

Andrew Murray
quoted hunk
+
 /*
  * iATU Unroll-specific register definitions
  * From 4.80 core version the address translation will be made by unroll
@@ -399,6 +404,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u8 interrupt_num);
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u16 interrupt_num);
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no,
+				       u16 interrupt_num);
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar);
 #else
 static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
@@ -431,6 +438,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 	return 0;
 }
 
+static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep,
+						     u8 func_no,
+						     u16 interrupt_num)
+{
+	return 0;
+}
+
 static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
 {
 }
-- 
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [PATCH 05/10] PCI: layerscape: Modify the way of getting capability with different PEX

From: Andrew Murray <hidden>
Date: 2019-08-15 12:51:10

On Thu, Aug 15, 2019 at 04:37:11PM +0800, Xiaowei Bao wrote:
quoted hunk
The different PCIe controller in one board may be have different
capability of MSI or MSIX, so change the way of getting the MSI
capability, make it more flexible.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 28 +++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index be61d96..9404ca0 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -22,6 +22,7 @@
 
 struct ls_pcie_ep {
 	struct dw_pcie		*pci;
+	struct pci_epc_features	*ls_epc;
 };
 
 #define to_ls_pcie_ep(x)	dev_get_drvdata((x)->dev)
@@ -40,25 +41,26 @@ static const struct of_device_id ls_pcie_ep_of_match[] = {
 	{ },
 };
 
-static const struct pci_epc_features ls_pcie_epc_features = {
-	.linkup_notifier = false,
-	.msi_capable = true,
-	.msix_capable = false,
-};
-
 static const struct pci_epc_features*
 ls_pcie_ep_get_features(struct dw_pcie_ep *ep)
 {
-	return &ls_pcie_epc_features;
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
+
+	return pcie->ls_epc;
 }
 
 static void ls_pcie_ep_init(struct dw_pcie_ep *ep)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
 	enum pci_barno bar;
 
 	for (bar = BAR_0; bar <= BAR_5; bar++)
 		dw_pcie_ep_reset_bar(pci, bar);
+
+	pcie->ls_epc->msi_capable = ep->msi_cap ? true : false;
+	pcie->ls_epc->msix_capable = ep->msix_cap ? true : false;
 }
 
 static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
@@ -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct dw_pcie *pci;
 	struct ls_pcie_ep *pcie;
+	struct pci_epc_features *ls_epc;
 	struct resource *dbi_base;
 	int ret;
 
@@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	if (!pci)
 		return -ENOMEM;
 
+	ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL);
+	if (!ls_epc)
+		return -ENOMEM;
+
 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
 	if (IS_ERR(pci->dbi_base))
@@ -139,6 +146,13 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev)
 	pci->ops = &ls_pcie_ep_ops;
 	pcie->pci = pci;
 
+	ls_epc->linkup_notifier = false,
+	ls_epc->msi_capable = true,
+	ls_epc->msix_capable = true,
As [msi,msix]_capable is shortly set from ls_pcie_ep_init - is there any
reason to set them here (to potentially incorrect values)?

Thanks,

Andrew Murray
+	ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4),
+
+	pcie->ls_epc = ls_epc;
+
 	platform_set_drvdata(pdev, pcie);
 
 	ret = ls_add_pcie_ep(pcie, pdev);
-- 
2.9.5

RE: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Xiaowei Bao <hidden>
Date: 2019-08-16 02:57:46

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:32
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for
DWC

On Thu, Aug 15, 2019 at 04:37:07PM +0800, Xiaowei Bao wrote:
quoted
Add multiple PFs support for DWC, different PF have different config
space, we use pf-offset property which get from the DTS to access the
different pF config space.
Thanks for the patch. I haven't seen a cover letter for this series, is there one
missing?
Maybe I miss, I will add you to review next time, thanks a lot for your comments.
quoted
Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97
+++++++++++++---------
quoted
 drivers/pci/controller/dwc/pcie-designware.c    | 105
++++++++++++++++++++++--
quoted
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
 	pci_epc_linkup(epc);
 }

-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno
bar,
quoted
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no,
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
I think I'd rather see this arithmetic (and the one for determining pf_base)
inside a macro or inline header function. This would make this code more
readable and reduce the chances of an error by avoiding duplication of code.

For example look at cdns_pcie_ep_fn_writeb and
ROCKCHIP_PCIE_EP_FUNC_BASE for examples of other EP drivers that do
this.
Agree, this looks fine, thanks a lot for your comments, I will use this way to access
the registers in next version patch.
quoted
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0);
@@ -37,7 +39,12 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie
*pci, enum pci_barno bar,

 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)  {
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }

 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,
@@ -78,28 +85,29 @@ static int dw_pcie_ep_write_header(struct pci_epc
*epc, u8 func_no,  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;

 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID, hdr->vendorid);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID, hdr->deviceid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID, hdr->revid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG, hdr->progif_code);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_VENDOR_ID,
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID,
hdr->subsys_id);
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);

 	return 0;
 }

-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum
pci_barno bar,
quoted
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)
 {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct
dw_pcie_ep *ep, enum pci_barno bar,
quoted
 		return -EINVAL;
 	}

-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar,
+cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n"); @@ -125,7
+133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep,
enum pci_barno bar,
quoted
 	return 0;
 }

-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t
phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)
 {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win,
PCIE_ATU_TYPE_MEM,
quoted
+				     phys_addr, pci_addr, size);

 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr; @@ -154,7 +163,7 @@
static
quoted
void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];

-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);

 	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
 	clear_bit(atu_index, ep->ib_window_map); @@ -170,14 +179,16 @@
static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;

 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;

-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type);
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc
*epc, u8 func_no,
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct pci_epc
*epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct pci_epc
*epc, u8 func_no, u8 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -287,12 +300,13
@@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct pci_epc
*epc, u8 func_no, u16 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep, u8 func_no,
quoted
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 		return -EINVAL;

 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); @@
-439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep,
u8 func_no,  }

 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc;
@@ -447,16 +463,17 @@ int dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;

-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
@@
quoted
-592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);

-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions", &epc->max_functions);
 	if (ret < 0)
 		epc->max_functions = 1;

+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
Bad things will likely happen if max_functions > 1 and pf-offset isn't set.
I think the driver should bail in this situation. It would be very easy for
someone to misconfigure this.
Yes, you are right, but if the max-functions have defined in DTS, require the pf-offset
must define in DTS, I am not sure the correct value of pf-offsetfor other platforms, 
so I think the max-functions and pf-offset should not have the dependence.
even though I didn't define pf-offset when I defined max-functions, the pf-offset is 0, 
the DWC ep driver can continue run the progress of INIT but not return, of course, 
thus the PF1 will not work, I don't know which way is better.
quoted
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c
b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void dw_pcie_writel_ob_unroll(struct
dw_pcie *pci, u32 index, u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

+static void dw_pcie_prog_ep_outbound_atu_unroll(struct dw_pcie *pci, u8
func_no,
quoted
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_BASE,
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_BASE,
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
+				 type | PCIE_ATU_FUNC_NUM(func_no));
With the exception of this line, the rest of this function is identical to
dw_pcie_prog_outbound_atu_unroll.
Yes, I can integrate the same code, but I think we'd better use the different outbound 
window set function between RC and EP, because the RC don't need the func_num parameter.
quoted
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int
index,
quoted
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size)
@@ -194,6 +231,51 @@ static void
dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");  }

+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no, index, type,
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
The same here, this is identical to dw_pcie_prog_outbound_atu with the
exception of this line.

Is there a way you can avoid all of this duplicated code?
As above, I can integrate the same code, but I keep to think the different outbound 
Window set function should be used between RC and EP.
Thanks,

Andrew Murray
quoted
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 			       u64 cpu_addr, u64 pci_addr, u32 size)  { @@ -252,8
+334,8
quoted
@@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index,
u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8
func_no,
quoted
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)  {
 	int type;
@@ -275,8 +357,10 @@ static int dw_pcie_prog_inbound_atu_unroll(struct
dw_pcie *pci, int index,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
type);
quoted
+	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
type |
quoted
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
@@ -297,14 +381,15 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	return -EBUSY;
 }

-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type)
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;

 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar,
 						       cpu_addr, as_type);

 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
PCIE_ATU_REGION_INBOUND |
quoted
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie
*pci, int index, int bar,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));

 	/*
 	 * Make sure ATU enable takes effect before any subsequent config
diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci);
void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type);
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);
 void dw_pcie_setup(struct dw_pcie *pci); diff --git
a/include/linux/pci-epc.h b/include/linux/pci-epc.h index
f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP controller */
 	spinlock_t			lock;
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Flists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=02%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C0e39168f6f144db6840308d721742040%7
C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C637014654998524452&amp;sd
ata=bP7eh
quoted
cjlGXCMVFE2b4f12Q6fGV7lQ%2F5i9qIi9FoPlbI%3D&amp;reserved=0

RE: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Xiaowei Bao <hidden>
Date: 2019-08-16 02:58:43

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:54
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of
MSI-X in EP mode

On Thu, Aug 15, 2019 at 04:37:08PM +0800, Xiaowei Bao wrote:
quoted
Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h    | 14 ++++++++++++++
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep, u8 func_no,
quoted
 	return 0;
 }

+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
Have I understood correctly that the hardware provides an alternative
mechanism that allows for raising MSI-X interrupts without the bother of
reading the capabilities registers?
Yes, the hardware provide two way to MSI-X, please check the page 492 of 
DWC_pcie_dm_registers_4.30 Menu.
MSIX_DOORBELL_OFF on page 492 0x948 Description: MSI-X Doorbell Register....>
If so is there any good reason to keep dw_pcie_ep_raise_msix_irq? (And thus
use it in dw_plat_pcie_ep_raise_irq also)?
I am not sure, but I think the dw_pcie_ep_raise_msix_irq function is not correct, 
because I think we can't get the MSIX table from the address ep->phys_base + tbl_addr, 
but I also don't know where I can get the correct MSIX table.
quoted
 {
diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index 2b291e8..cd903e9 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -88,6 +88,11 @@
 #define PCIE_MISC_CONTROL_1_OFF		0x8BC
 #define PCIE_DBI_RO_WR_EN		BIT(0)

+#define PCIE_MSIX_DOORBELL		0x948
+#define PCIE_MSIX_DOORBELL_PF_SHIFT	24
+#define PCIE_MSIX_DOORBELL_VF_SHIFT	16
+#define PCIE_MSIX_DOORBELL_VF_ACTIVE	BIT(15)
The _VF defines are not used, I'd suggest removing them.
In fact, I will add the SRIOV support in this file, the SRIOV feature have verified 
In my board, but I need wait the EP framework SRIOV patch merge, 
so I defined these two macros.
Thanks,

Andrew Murray
quoted
+
 /*
  * iATU Unroll-specific register definitions
  * From 4.80 core version the address translation will be made by
unroll @@ -399,6 +404,8 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 			     u8 interrupt_num);
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u16 interrupt_num);
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num);
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar);
#else  static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) @@
-431,6 +438,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	return 0;
 }

+static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep
*ep,
quoted
+						     u8 func_no,
+						     u16 interrupt_num)
+{
+	return 0;
+}
+
 static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno bar)  {  }
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Flists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=02%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C8489493003bb48a0139d08d721773972%
7C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C637014668369499298&amp;sd
ata=dyrXB
quoted
avljJBFUSNXW7K%2FRoXvwfWTE%2FoU2KMd1bZkJow%3D&amp;reserved=0

RE: [PATCH 05/10] PCI: layerscape: Modify the way of getting capability with different PEX

From: Xiaowei Bao <hidden>
Date: 2019-08-16 03:00:19

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 20:51
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 05/10] PCI: layerscape: Modify the way of getting
capability with different PEX

On Thu, Aug 15, 2019 at 04:37:11PM +0800, Xiaowei Bao wrote:
quoted
The different PCIe controller in one board may be have different
capability of MSI or MSIX, so change the way of getting the MSI
capability, make it more flexible.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 28
+++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c
b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index be61d96..9404ca0 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -22,6 +22,7 @@

 struct ls_pcie_ep {
 	struct dw_pcie		*pci;
+	struct pci_epc_features	*ls_epc;
 };

 #define to_ls_pcie_ep(x)	dev_get_drvdata((x)->dev)
@@ -40,25 +41,26 @@ static const struct of_device_id
ls_pcie_ep_of_match[] = {
quoted
 	{ },
 };

-static const struct pci_epc_features ls_pcie_epc_features = {
-	.linkup_notifier = false,
-	.msi_capable = true,
-	.msix_capable = false,
-};
-
 static const struct pci_epc_features*  ls_pcie_ep_get_features(struct
dw_pcie_ep *ep)  {
-	return &ls_pcie_epc_features;
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
+
+	return pcie->ls_epc;
 }

 static void ls_pcie_ep_init(struct dw_pcie_ep *ep)  {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
 	enum pci_barno bar;

 	for (bar = BAR_0; bar <= BAR_5; bar++)
 		dw_pcie_ep_reset_bar(pci, bar);
+
+	pcie->ls_epc->msi_capable = ep->msi_cap ? true : false;
+	pcie->ls_epc->msix_capable = ep->msix_cap ? true : false;
 }

 static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, @@
-118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device
*pdev)
quoted
 	struct device *dev = &pdev->dev;
 	struct dw_pcie *pci;
 	struct ls_pcie_ep *pcie;
+	struct pci_epc_features *ls_epc;
 	struct resource *dbi_base;
 	int ret;
@@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct
platform_device *pdev)
quoted
 	if (!pci)
 		return -ENOMEM;

+	ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL);
+	if (!ls_epc)
+		return -ENOMEM;
+
 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"regs");
quoted
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
 	if (IS_ERR(pci->dbi_base))
@@ -139,6 +146,13 @@ static int __init ls_pcie_ep_probe(struct
platform_device *pdev)
quoted
 	pci->ops = &ls_pcie_ep_ops;
 	pcie->pci = pci;

+	ls_epc->linkup_notifier = false,
+	ls_epc->msi_capable = true,
+	ls_epc->msix_capable = true,
As [msi,msix]_capable is shortly set from ls_pcie_ep_init - is there any reason
to set them here (to potentially incorrect values)?
This is a INIT value, maybe false is better for msi_capable and msix_capable, 
of course, we don't need to set it.
Thanks,

Andrew Murray
quoted
+	ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4),
+
+	pcie->ls_epc = ls_epc;
+
 	platform_set_drvdata(pdev, pcie);

 	ret = ls_add_pcie_ep(pcie, pdev);
--
2.9.5

Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Andrew Murray <hidden>
Date: 2019-08-16 09:44:56

On Fri, Aug 16, 2019 at 02:55:41AM +0000, Xiaowei Bao wrote:
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:32
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for
DWC

On Thu, Aug 15, 2019 at 04:37:07PM +0800, Xiaowei Bao wrote:
quoted
Add multiple PFs support for DWC, different PF have different config
space, we use pf-offset property which get from the DTS to access the
different pF config space.
Thanks for the patch. I haven't seen a cover letter for this series, is there one
missing?
Maybe I miss, I will add you to review next time, thanks a lot for your comments.
quoted
quoted
Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97
+++++++++++++---------
quoted
 drivers/pci/controller/dwc/pcie-designware.c    | 105
++++++++++++++++++++++--
quoted
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep)
 	pci_epc_linkup(epc);
 }

-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno
bar,
quoted
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no,
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
I think I'd rather see this arithmetic (and the one for determining pf_base)
inside a macro or inline header function. This would make this code more
readable and reduce the chances of an error by avoiding duplication of code.

For example look at cdns_pcie_ep_fn_writeb and
ROCKCHIP_PCIE_EP_FUNC_BASE for examples of other EP drivers that do
this.
Agree, this looks fine, thanks a lot for your comments, I will use this way to access
the registers in next version patch.
quoted
quoted
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0);
@@ -37,7 +39,12 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie
*pci, enum pci_barno bar,

 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)  {
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }

 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr,
@@ -78,28 +85,29 @@ static int dw_pcie_ep_write_header(struct pci_epc
*epc, u8 func_no,  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;

 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID, hdr->vendorid);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID, hdr->deviceid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID, hdr->revid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG, hdr->progif_code);
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_VENDOR_ID,
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID,
hdr->subsys_id);
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);

 	return 0;
 }

-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum
pci_barno bar,
quoted
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)
 {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct
dw_pcie_ep *ep, enum pci_barno bar,
quoted
 		return -EINVAL;
 	}

-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar,
+cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n"); @@ -125,7
+133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep,
enum pci_barno bar,
quoted
 	return 0;
 }

-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t
phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no,
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)
 {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win,
PCIE_ATU_TYPE_MEM,
quoted
+				     phys_addr, pci_addr, size);

 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr; @@ -154,7 +163,7 @@
static
quoted
void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];

-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);

 	dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND);
 	clear_bit(atu_index, ep->ib_window_map); @@ -170,14 +179,16 @@
static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;

 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;

-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type);
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc
*epc, u8 func_no,
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct pci_epc
*epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct pci_epc
*epc, u8 func_no, u8 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -287,12 +300,13
@@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct pci_epc
*epc, u8 func_no, u16 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep, u8 func_no,
quoted
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 		return -EINVAL;

 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); @@
-439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep,
u8 func_no,  }

 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc;
@@ -447,16 +463,17 @@ int dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;

-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK);
@@
quoted
-592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);

-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions", &epc->max_functions);
 	if (ret < 0)
 		epc->max_functions = 1;

+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
Bad things will likely happen if max_functions > 1 and pf-offset isn't set.
I think the driver should bail in this situation. It would be very easy for
someone to misconfigure this.
Yes, you are right, but if the max-functions have defined in DTS, require the pf-offset
must define in DTS, I am not sure the correct value of pf-offsetfor other platforms, 
so I think the max-functions and pf-offset should not have the dependence.
Yes you're correct. I hadn't really thought about this beyond layerscape. It's
also possible that other hardware could support multiple PFs without relying on
an offset and perhaps employ some other mechanism to access different
functions. So whilst this property can be optional for the majority of dwc
controllers - it must be set and cannot be zero for layerscape.

Perhaps inside ls_pcie_ep_init, you can set max_functions to 1 if pf_offset is
0 and print a WARN to explain why? (Or ls_pcie_ep_init returns failure and
dw_pcie_ep_init checks it and bails).

The assumption is being made here that future dw controllers may also use
pf_offset (is this likely?) - otherwise why is this in pcie-designware-ep.c and
not pci-layerscape-ep.c and why is this value not just hard-coded for lp?

even though I didn't define pf-offset when I defined max-functions, the pf-offset is 0, 
the DWC ep driver can continue run the progress of INIT but not return, of course, 
thus the PF1 will not work, I don't know which way is better.
quoted
quoted
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c
b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void dw_pcie_writel_ob_unroll(struct
dw_pcie *pci, u32 index, u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

+static void dw_pcie_prog_ep_outbound_atu_unroll(struct dw_pcie *pci, u8
func_no,
quoted
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_BASE,
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_BASE,
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LOWER_TARGET,
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET,
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
+				 type | PCIE_ATU_FUNC_NUM(func_no));
With the exception of this line, the rest of this function is identical to
dw_pcie_prog_outbound_atu_unroll.
Yes, I can integrate the same code, but I think we'd better use the different outbound 
window set function between RC and EP, because the RC don't need the func_num parameter.

quoted
quoted
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int
index,
quoted
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size)
@@ -194,6 +231,51 @@ static void
dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");  }

+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no, index, type,
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
The same here, this is identical to dw_pcie_prog_outbound_atu with the
exception of this line.

Is there a way you can avoid all of this duplicated code?
As above, I can integrate the same code, but I keep to think the different outbound 
Window set function should be used between RC and EP.
Or, is it possible to keep and use the existing functions, but use them
differently, e.g:

@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr,
                return -EINVAL;
        }

-       dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-                                 phys_addr, pci_addr, size);
+       dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM_FUNC(func_no),
+                                    phys_addr, pci_addr, size);

        set_bit(free_win, ep->ob_window_map);
        ep->outbound_addr[free_win] = phys_addr;

Supported with:

#define PCIE_ATU_TYPE_MEM               0x0
#define PCIE_ATU_TYPE_MEM_FUNC(func_no) (PCIE_ATU_TYPE_MEM | PCIE_ATU_FUNC_NUM(func_no))


This is just a suggestion, but I'm keen to avoid code duplication.
quoted
Thanks,

Andrew Murray
quoted
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) {
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type,
 			       u64 cpu_addr, u64 pci_addr, u32 size)  { @@ -252,8
+334,8
quoted
@@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index,
u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8
func_no,
quoted
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)  {
 	int type;
@@ -275,8 +357,10 @@ static int dw_pcie_prog_inbound_atu_unroll(struct
dw_pcie *pci, int index,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
type);
quoted
+	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1,
type |
quoted
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2,
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
@@ -297,14 +381,15 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	return -EBUSY;
 }

-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type)
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;

 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar,
 						       cpu_addr, as_type);

 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
PCIE_ATU_REGION_INBOUND |
quoted
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie
*pci, int index, int bar,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));

 	/*
 	 * Make sure ATU enable takes effect before any subsequent config
diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci);
void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type);
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);
 void dw_pcie_setup(struct dw_pcie *pci); diff --git
a/include/linux/pci-epc.h b/include/linux/pci-epc.h index
f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
Also pf_offset is an implementation detail needed only by the driver to
calculate where the PF is - it doesn't seem right that we share this with the
EP controller framework (whereas max_functions is used as a bounds check
for func_no in the framework calls).

I'd suggest that pf_offset is moved to a dwc structure, perhaps dw_pcie_ep?

Thanks,

Andrew Murray
quoted
quoted
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP controller */
 	spinlock_t			lock;
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Flists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=02%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C0e39168f6f144db6840308d721742040%7
C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C637014654998524452&amp;sd
ata=bP7eh
quoted
cjlGXCMVFE2b4f12Q6fGV7lQ%2F5i9qIi9FoPlbI%3D&amp;reserved=0

Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Andrew Murray <hidden>
Date: 2019-08-16 10:20:30

On Fri, Aug 16, 2019 at 02:58:31AM +0000, Xiaowei Bao wrote:
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:54
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of
MSI-X in EP mode

On Thu, Aug 15, 2019 at 04:37:08PM +0800, Xiaowei Bao wrote:
quoted
Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h    | 14 ++++++++++++++
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep, u8 func_no,
quoted
 	return 0;
 }

+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
Have I understood correctly that the hardware provides an alternative
mechanism that allows for raising MSI-X interrupts without the bother of
reading the capabilities registers?
Yes, the hardware provide two way to MSI-X, please check the page 492 of 
DWC_pcie_dm_registers_4.30 Menu.
MSIX_DOORBELL_OFF on page 492 0x948 Description: MSI-X Doorbell Register....>
Thanks for the reference.
quoted
If so is there any good reason to keep dw_pcie_ep_raise_msix_irq? (And thus
use it in dw_plat_pcie_ep_raise_irq also)?
I am not sure, but I think the dw_pcie_ep_raise_msix_irq function is not correct, 
because I think we can't get the MSIX table from the address ep->phys_base + tbl_addr, 
but I also don't know where I can get the correct MSIX table.
Well it looks like this function is used by snps,dw-pcie-ep and snps,dw-pcie,
perhaps the doorbell mode isn't available on that hardware.
quoted
quoted
 {
diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index 2b291e8..cd903e9 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -88,6 +88,11 @@
 #define PCIE_MISC_CONTROL_1_OFF		0x8BC
 #define PCIE_DBI_RO_WR_EN		BIT(0)

+#define PCIE_MSIX_DOORBELL		0x948
+#define PCIE_MSIX_DOORBELL_PF_SHIFT	24
+#define PCIE_MSIX_DOORBELL_VF_SHIFT	16
+#define PCIE_MSIX_DOORBELL_VF_ACTIVE	BIT(15)
The _VF defines are not used, I'd suggest removing them.
In fact, I will add the SRIOV support in this file, the SRIOV feature have verified 
In my board, but I need wait the EP framework SRIOV patch merge, 
so I defined these two macros.
I'd suggest adding the VF macros along with the SRIOV feature.

Thanks,

Andrew Murray
quoted
Thanks,

Andrew Murray
quoted
+
 /*
  * iATU Unroll-specific register definitions
  * From 4.80 core version the address translation will be made by
unroll @@ -399,6 +404,8 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 			     u8 interrupt_num);
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u16 interrupt_num);
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num);
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar);
#else  static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) @@
-431,6 +438,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	return 0;
 }

+static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep
*ep,
quoted
+						     u8 func_no,
+						     u16 interrupt_num)
+{
+	return 0;
+}
+
 static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno bar)  {  }
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
https://eur01.safelinks.protection.outlook.com/?url=http%3A%2F%2Flists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=02%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C8489493003bb48a0139d08d721773972%
7C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C637014668369499298&amp;sd
ata=dyrXB
quoted
avljJBFUSNXW7K%2FRoXvwfWTE%2FoU2KMd1bZkJow%3D&amp;reserved=0

Re: [PATCH 05/10] PCI: layerscape: Modify the way of getting capability with different PEX

From: Andrew Murray <hidden>
Date: 2019-08-16 10:25:50

On Fri, Aug 16, 2019 at 03:00:00AM +0000, Xiaowei Bao wrote:
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 20:51
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 05/10] PCI: layerscape: Modify the way of getting
capability with different PEX

On Thu, Aug 15, 2019 at 04:37:11PM +0800, Xiaowei Bao wrote:
quoted
The different PCIe controller in one board may be have different
capability of MSI or MSIX, so change the way of getting the MSI
capability, make it more flexible.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 28
+++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c
b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index be61d96..9404ca0 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -22,6 +22,7 @@

 struct ls_pcie_ep {
 	struct dw_pcie		*pci;
+	struct pci_epc_features	*ls_epc;
 };

 #define to_ls_pcie_ep(x)	dev_get_drvdata((x)->dev)
@@ -40,25 +41,26 @@ static const struct of_device_id
ls_pcie_ep_of_match[] = {
quoted
 	{ },
 };

-static const struct pci_epc_features ls_pcie_epc_features = {
-	.linkup_notifier = false,
-	.msi_capable = true,
-	.msix_capable = false,
-};
-
 static const struct pci_epc_features*  ls_pcie_ep_get_features(struct
dw_pcie_ep *ep)  {
-	return &ls_pcie_epc_features;
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
+
+	return pcie->ls_epc;
 }

 static void ls_pcie_ep_init(struct dw_pcie_ep *ep)  {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
 	enum pci_barno bar;

 	for (bar = BAR_0; bar <= BAR_5; bar++)
 		dw_pcie_ep_reset_bar(pci, bar);
+
+	pcie->ls_epc->msi_capable = ep->msi_cap ? true : false;
+	pcie->ls_epc->msix_capable = ep->msix_cap ? true : false;
 }

 static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, @@
-118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device
*pdev)
quoted
 	struct device *dev = &pdev->dev;
 	struct dw_pcie *pci;
 	struct ls_pcie_ep *pcie;
+	struct pci_epc_features *ls_epc;
 	struct resource *dbi_base;
 	int ret;
@@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct
platform_device *pdev)
quoted
 	if (!pci)
 		return -ENOMEM;

+	ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL);
+	if (!ls_epc)
+		return -ENOMEM;
+
 	dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM,
"regs");
quoted
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
 	if (IS_ERR(pci->dbi_base))
@@ -139,6 +146,13 @@ static int __init ls_pcie_ep_probe(struct
platform_device *pdev)
quoted
 	pci->ops = &ls_pcie_ep_ops;
 	pcie->pci = pci;

+	ls_epc->linkup_notifier = false,
+	ls_epc->msi_capable = true,
+	ls_epc->msix_capable = true,
As [msi,msix]_capable is shortly set from ls_pcie_ep_init - is there any reason
to set them here (to potentially incorrect values)?
This is a INIT value, maybe false is better for msi_capable and msix_capable, 
of course, we don't need to set it.
ls_epc is kzalloc'd and so all zeros, so you get false for free. I think you
can remove these two lines (or all three if you don't care that linkup_notifier
isn't explicitly set).

Thanks,

Andrew Murray
quoted
Thanks,

Andrew Murray
quoted
+	ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4),
+
+	pcie->ls_epc = ls_epc;
+
 	platform_set_drvdata(pdev, pcie);

 	ret = ls_add_pcie_ep(pcie, pdev);
--
2.9.5

Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Kishon Vijay Abraham I <hidden>
Date: 2019-08-16 10:52:11

Hi,

On 16/08/19 8:28 AM, Xiaowei Bao wrote:
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:54
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of
MSI-X in EP mode

On Thu, Aug 15, 2019 at 04:37:08PM +0800, Xiaowei Bao wrote:
quoted
Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++
 drivers/pci/controller/dwc/pcie-designware.h    | 14 ++++++++++++++
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep, u8 func_no,
quoted
 	return 0;
 }

+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
Have I understood correctly that the hardware provides an alternative
mechanism that allows for raising MSI-X interrupts without the bother of
reading the capabilities registers?
Yes, the hardware provide two way to MSI-X, please check the page 492 of 
DWC_pcie_dm_registers_4.30 Menu.
MSIX_DOORBELL_OFF on page 492 0x948 Description: MSI-X Doorbell Register....>
quoted
If so is there any good reason to keep dw_pcie_ep_raise_msix_irq? (And thus
use it in dw_plat_pcie_ep_raise_irq also)?
I am not sure, but I think the dw_pcie_ep_raise_msix_irq function is not correct, 
because I think we can't get the MSIX table from the address ep->phys_base + tbl_addr, 
but I also don't know where I can get the correct MSIX table.
Sometime back when I tried raising MSI-X from EP, it was failing. It's quite
possible dw_pcie_ep_raise_msix_irq function is not correct.

MSI-X table can be obtained from the inbound ATU corresponding to the MSIX bar.
IMO MSI-X support in EP mode needs rework. For instance set_msix should also
take BAR number as input to be configured in the MSI-X capability. The function
driver (pci-epf-test.c) should allocate memory taking into account the MSI-X table.

Thanks
Kishon

RE: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Xiaowei Bao <hidden>
Date: 2019-08-16 11:00:23

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月16日 17:45
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
mark.rutland@arm.com; shawnguo@kernel.org; Leo Li
[off-list ref]; kishon@ti.com; lorenzo.pieralisi@arm.com;
arnd@arndb.de; gregkh@linuxfoundation.org; M.h. Lian
[off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org; Z.q. Hou [off-list ref]
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for
DWC

On Fri, Aug 16, 2019 at 02:55:41AM +0000, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:32
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de;
gregkh@linuxfoundation.org; M.h. Lian [off-list ref];
Mingkai Hu [off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs
support for DWC

On Thu, Aug 15, 2019 at 04:37:07PM +0800, Xiaowei Bao wrote:
quoted
Add multiple PFs support for DWC, different PF have different
config space, we use pf-offset property which get from the DTS to
access the different pF config space.
Thanks for the patch. I haven't seen a cover letter for this series,
is there one missing?
Maybe I miss, I will add you to review next time, thanks a lot for your
comments.
quoted
quoted
quoted
Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97
+++++++++++++---------
quoted
 drivers/pci/controller/dwc/pcie-designware.c    | 105
++++++++++++++++++++++--
quoted
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep
*ep)
quoted
quoted
quoted
 	pci_epc_linkup(epc);
 }

-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno
bar,
quoted
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no,
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
I think I'd rather see this arithmetic (and the one for determining
pf_base) inside a macro or inline header function. This would make
this code more readable and reduce the chances of an error by avoiding
duplication of code.
quoted
quoted
For example look at cdns_pcie_ep_fn_writeb and
ROCKCHIP_PCIE_EP_FUNC_BASE for examples of other EP drivers that do
this.
Agree, this looks fine, thanks a lot for your comments, I will use
this way to access the registers in next version patch.
quoted
quoted
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0); @@ -37,7 +39,12 @@ static
void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno
bar,

 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
{
quoted
quoted
quoted
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }

 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8
cap_ptr, @@ -78,28 +85,29 @@ static int
dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;

 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID,
hdr->vendorid);
quoted
quoted
quoted
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID, hdr->deviceid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID, hdr->revid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG,
hdr->progif_code);
quoted
quoted
quoted
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_VENDOR_ID,
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID,
hdr->subsys_id);
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);

 	return 0;
 }

-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum
pci_barno bar,
quoted
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)  {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct
dw_pcie_ep *ep, enum pci_barno bar,
quoted
 		return -EINVAL;
 	}

-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar,
+cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n"); @@
-125,7
quoted
quoted
quoted
+133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep,
enum pci_barno bar,
quoted
 	return 0;
 }

-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep,
phys_addr_t phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)  {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win,
PCIE_ATU_TYPE_MEM,
quoted
+				     phys_addr, pci_addr, size);

 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr; @@ -154,7 +163,7
@@
quoted
quoted
static
quoted
void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];

-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);

 	dw_pcie_disable_atu(pci, atu_index,
DW_PCIE_REGION_INBOUND);
quoted
quoted
quoted
 	clear_bit(atu_index, ep->ib_window_map); @@ -170,14 +179,16
@@
quoted
quoted
quoted
static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;

 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;

-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr,
as_type);
quoted
quoted
quoted
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc
*epc, u8 func_no,
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr,
+size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct pci_epc
*epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct pci_epc
*epc, u8 func_no, u8 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -287,12
+300,13 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8
func_no)  {
quoted
quoted
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct
pci_epc *epc, u8 func_no, u16 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep
quoted
quoted
*ep, u8 func_no,
quoted
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 		return -EINVAL;

 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1.
*/
quoted
quoted
quoted
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
@@
quoted
quoted
quoted
-439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep,
u8 func_no,  }

 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc;
@@ -447,16 +463,17 @@ int dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;

-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower &
PCI_BASE_ADDRESS_MEM_TYPE_MASK);
quoted
quoted
@@
quoted
-592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);

-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions",
&epc->max_functions);
quoted
quoted
quoted
 	if (ret < 0)
 		epc->max_functions = 1;

+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
Bad things will likely happen if max_functions > 1 and pf-offset isn't set.
I think the driver should bail in this situation. It would be very
easy for someone to misconfigure this.
Yes, you are right, but if the max-functions have defined in DTS,
require the pf-offset must define in DTS, I am not sure the correct
value of pf-offsetfor other platforms, so I think the max-functions and
pf-offset should not have the dependence.

Yes you're correct. I hadn't really thought about this beyond layerscape. It's
also possible that other hardware could support multiple PFs without relying
on an offset and perhaps employ some other mechanism to access different
functions. So whilst this property can be optional for the majority of dwc
controllers - it must be set and cannot be zero for layerscape.

Perhaps inside ls_pcie_ep_init, you can set max_functions to 1 if pf_offset is
0 and print a WARN to explain why? (Or ls_pcie_ep_init returns failure and
dw_pcie_ep_init checks it and bails).

The assumption is being made here that future dw controllers may also use
pf_offset (is this likely?) - otherwise why is this in pcie-designware-ep.c and
not pci-layerscape-ep.c and why is this value not just hard-coded for lp?
Thanks a lot for your detail comments, this give me a lot of help.
Yes, I agree your point, and I will seriously consider a best way to fix this potential issue.
Based on your experience, how do other platforms implement the multiple functions?
The DWC core difference the different PF by signal "client0_tlp_func_num[(PF_WD-1):0]"
quoted
even though I didn't define pf-offset when I defined max-functions,
the pf-offset is 0, the DWC ep driver can continue run the progress of
INIT but not return, of course, thus the PF1 will not work, I don't know which
way is better.
Hi Andrew,
quoted hunk
quoted
quoted
quoted
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c
b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void dw_pcie_writel_ob_unroll(struct
dw_pcie *pci, u32 index, u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

+static void dw_pcie_prog_ep_outbound_atu_unroll(struct dw_pcie
+*pci, u8
func_no,
quoted
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_LOWER_BASE,
quoted
quoted
quoted
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_UPPER_BASE,
quoted
quoted
quoted
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_LOWER_TARGET,
quoted
quoted
quoted
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_UPPER_TARGET,
quoted
quoted
quoted
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
quoted
+				 type | PCIE_ATU_FUNC_NUM(func_no));
With the exception of this line, the rest of this function is
identical to dw_pcie_prog_outbound_atu_unroll.
Yes, I can integrate the same code, but I think we'd better use the
different outbound window set function between RC and EP, because the RC
don't need the func_num parameter.


quoted
quoted
quoted
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL2,
quoted
quoted
quoted
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++)
{
quoted
quoted
quoted
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci,
int
index,
quoted
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size) @@ -194,6
+231,51 @@ static
quoted
quoted
quoted
void
dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");  }

+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8
+func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no, index,
type,
quoted
quoted
quoted
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
The same here, this is identical to dw_pcie_prog_outbound_atu with
the exception of this line.

Is there a way you can avoid all of this duplicated code?
As above, I can integrate the same code, but I keep to think the
different outbound Window set function should be used between RC and EP.
Or, is it possible to keep and use the existing functions, but use them
differently, e.g:

@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                return -EINVAL;
        }

-       dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-                                 phys_addr, pci_addr, size);
+       dw_pcie_prog_outbound_atu(pci, free_win,
PCIE_ATU_TYPE_MEM_FUNC(func_no),
+                                    phys_addr, pci_addr, size);

        set_bit(free_win, ep->ob_window_map);
        ep->outbound_addr[free_win] = phys_addr;


Supported with:

#define PCIE_ATU_TYPE_MEM               0x0
#define PCIE_ATU_TYPE_MEM_FUNC(func_no) (PCIE_ATU_TYPE_MEM |
PCIE_ATU_FUNC_NUM(func_no))


This is just a suggestion, but I'm keen to avoid code duplication.
Thanks, I have think of a way as follow:

This is a good way, but I think PCIE_ATU_TYPE_MEM_FUNC(func_no) will give
Someone confused meaning, because PCIE_ATU_TYPE_MEM indicate the type of TLP,
and the location in the bit[0:3] of register CR1, but the PCIE_ATU_FUNC_NUM is bit[20:24],
I have another way:
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                 return -EINVAL;
        }
		
		dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
                                phys_addr, pci_addr, size);
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR1);
+		dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val | PCIE_ATU_FUNC_NUM(func_no));
or 
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+                                   int type, u64 cpu_addr, u64 pci_addr,
+                                   u32 size)
+{
+		dw_pcie_prog_outbound_atu(pci, index, type, cpu_addr, pci_addr, size);	
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR1);
+		dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val | PCIE_ATU_FUNC_NUM(func_no));
+}

Which do you think is better of these three ways?
quoted
quoted
Thanks,

Andrew Murray
quoted
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++)
{
quoted
quoted
quoted
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int
type,
quoted
quoted
quoted
 			       u64 cpu_addr, u64 pci_addr, u32 size)  { @@
-252,8
quoted
quoted
+334,8
quoted
@@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32
index,
u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int
index,
quoted
quoted
quoted
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci,
+u8
func_no,
quoted
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)  {
 	int type;
@@ -275,8 +357,10 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct
dw_pcie *pci, int index,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
type);
quoted
+	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
type |
quoted
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL2,
quoted
quoted
quoted
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
@@ -297,14 +381,15 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	return -EBUSY;
 }

-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type)
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
quoted
quoted
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;

 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index,
+bar,
 						       cpu_addr, as_type);

 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
PCIE_ATU_REGION_INBOUND |
quoted
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie
*pci, int index, int bar,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));

 	/*
 	 * Make sure ATU enable takes effect before any subsequent
config diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie
*pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type);
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8
+func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
quoted
quoted
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);  void
dw_pcie_setup(struct
quoted
quoted
quoted
dw_pcie *pci); diff --git a/include/linux/pci-epc.h
b/include/linux/pci-epc.h index f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
Also pf_offset is an implementation detail needed only by the driver to
calculate where the PF is - it doesn't seem right that we share this with the EP
controller framework (whereas max_functions is used as a bounds check for
func_no in the framework calls).

I'd suggest that pf_offset is moved to a dwc structure, perhaps dw_pcie_ep?
I add the variable to this struct is consider that all PF is belong to a PCIe controller,
and the pci_epc indicate a PCIe controller, so I add this variable to this struct, what
do you think about this? I am not sure whether I should add this variable to dw_pcie_ep.
Thanks,

Andrew Murray
quoted
quoted
quoted
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP controller */
 	spinlock_t			lock;
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=0
2
quoted
quoted
quoted
%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C0e39168f6f144db6840308d721742040%7
quoted
quoted
C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C637014654998524452&amp;sd
quoted
quoted
ata=bP7eh
quoted
cjlGXCMVFE2b4f12Q6fGV7lQ%2F5i9qIi9FoPlbI%3D&amp;reserved=0

RE: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Xiaowei Bao <hidden>
Date: 2019-08-16 11:01:37

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月16日 18:20
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org; Z.q. Hou [off-list ref]
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of
MSI-X in EP mode

On Fri, Aug 16, 2019 at 02:58:31AM +0000, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:54
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de;
gregkh@linuxfoundation.org; M.h. Lian [off-list ref];
Mingkai Hu [off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode
of MSI-X in EP mode

On Thu, Aug 15, 2019 at 04:37:08PM +0800, Xiaowei Bao wrote:
quoted
Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14
++++++++++++++
quoted
quoted
quoted
 drivers/pci/controller/dwc/pcie-designware.h    | 14
++++++++++++++
quoted
quoted
quoted
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep
*ep, u8 func_no,
quoted
 	return 0;
 }

+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
Have I understood correctly that the hardware provides an
alternative mechanism that allows for raising MSI-X interrupts
without the bother of reading the capabilities registers?
Yes, the hardware provide two way to MSI-X, please check the page 492
of
DWC_pcie_dm_registers_4.30 Menu.
MSIX_DOORBELL_OFF on page 492 0x948 Description: MSI-X Doorbell
Register....>
Thanks for the reference.
quoted
quoted
If so is there any good reason to keep dw_pcie_ep_raise_msix_irq?
(And thus use it in dw_plat_pcie_ep_raise_irq also)?
I am not sure, but I think the dw_pcie_ep_raise_msix_irq function is
not correct, because I think we can't get the MSIX table from the
address ep->phys_base + tbl_addr, but I also don't know where I can get the
correct MSIX table.

Well it looks like this function is used by snps,dw-pcie-ep and snps,dw-pcie,
perhaps the doorbell mode isn't available on that hardware.
quoted
quoted
quoted
 {
diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index 2b291e8..cd903e9 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -88,6 +88,11 @@
 #define PCIE_MISC_CONTROL_1_OFF		0x8BC
 #define PCIE_DBI_RO_WR_EN		BIT(0)

+#define PCIE_MSIX_DOORBELL		0x948
+#define PCIE_MSIX_DOORBELL_PF_SHIFT	24
+#define PCIE_MSIX_DOORBELL_VF_SHIFT	16
+#define PCIE_MSIX_DOORBELL_VF_ACTIVE	BIT(15)
The _VF defines are not used, I'd suggest removing them.
In fact, I will add the SRIOV support in this file, the SRIOV feature
have verified In my board, but I need wait the EP framework SRIOV
patch merge, so I defined these two macros.
I'd suggest adding the VF macros along with the SRIOV feature.
OK, I will remove these two macros. Thanks.
Thanks,

Andrew Murray
quoted
quoted
Thanks,

Andrew Murray
quoted
+
 /*
  * iATU Unroll-specific register definitions
  * From 4.80 core version the address translation will be made by
unroll @@ -399,6 +404,8 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 			     u8 interrupt_num);
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			     u16 interrupt_num);
+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num);
 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno
bar); #else  static inline void dw_pcie_ep_linkup(struct
dw_pcie_ep *ep) @@
-431,6 +438,13 @@ static inline int
dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	return 0;
 }

+static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct
+dw_pcie_ep
*ep,
quoted
+						     u8 func_no,
+						     u16 interrupt_num)
+{
+	return 0;
+}
+
 static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno bar)  {  }
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=0
2
quoted
quoted
quoted
%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C8489493003bb48a0139d08d721773972%
quoted
quoted
7C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C637014668369499298&amp;sd
quoted
quoted
ata=dyrXB
quoted
avljJBFUSNXW7K%2FRoXvwfWTE%2FoU2KMd1bZkJow%3D&amp;reserved=0

RE: [PATCH 05/10] PCI: layerscape: Modify the way of getting capability with different PEX

From: Xiaowei Bao <hidden>
Date: 2019-08-16 11:03:39

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月16日 18:26
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org; Z.q. Hou
[off-list ref]
Subject: Re: [PATCH 05/10] PCI: layerscape: Modify the way of getting
capability with different PEX

On Fri, Aug 16, 2019 at 03:00:00AM +0000, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 20:51
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de;
gregkh@linuxfoundation.org; M.h. Lian [off-list ref];
Mingkai Hu [off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 05/10] PCI: layerscape: Modify the way of
getting capability with different PEX

On Thu, Aug 15, 2019 at 04:37:11PM +0800, Xiaowei Bao wrote:
quoted
The different PCIe controller in one board may be have different
capability of MSI or MSIX, so change the way of getting the MSI
capability, make it more flexible.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pci-layerscape-ep.c | 28
+++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c
b/drivers/pci/controller/dwc/pci-layerscape-ep.c
index be61d96..9404ca0 100644
--- a/drivers/pci/controller/dwc/pci-layerscape-ep.c
+++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c
@@ -22,6 +22,7 @@

 struct ls_pcie_ep {
 	struct dw_pcie		*pci;
+	struct pci_epc_features	*ls_epc;
 };

 #define to_ls_pcie_ep(x)	dev_get_drvdata((x)->dev)
@@ -40,25 +41,26 @@ static const struct of_device_id
ls_pcie_ep_of_match[] = {
quoted
 	{ },
 };

-static const struct pci_epc_features ls_pcie_epc_features = {
-	.linkup_notifier = false,
-	.msi_capable = true,
-	.msix_capable = false,
-};
-
 static const struct pci_epc_features*
ls_pcie_ep_get_features(struct dw_pcie_ep *ep)  {
-	return &ls_pcie_epc_features;
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
+
+	return pcie->ls_epc;
 }

 static void ls_pcie_ep_init(struct dw_pcie_ep *ep)  {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci);
 	enum pci_barno bar;

 	for (bar = BAR_0; bar <= BAR_5; bar++)
 		dw_pcie_ep_reset_bar(pci, bar);
+
+	pcie->ls_epc->msi_capable = ep->msi_cap ? true : false;
+	pcie->ls_epc->msix_capable = ep->msix_cap ? true : false;
 }

 static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8
func_no, @@
-118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct
platform_device
*pdev)
quoted
 	struct device *dev = &pdev->dev;
 	struct dw_pcie *pci;
 	struct ls_pcie_ep *pcie;
+	struct pci_epc_features *ls_epc;
 	struct resource *dbi_base;
 	int ret;
@@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct
platform_device *pdev)
quoted
 	if (!pci)
 		return -ENOMEM;

+	ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL);
+	if (!ls_epc)
+		return -ENOMEM;
+
 	dbi_base = platform_get_resource_byname(pdev,
IORESOURCE_MEM,
quoted
quoted
"regs");
quoted
 	pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base);
 	if (IS_ERR(pci->dbi_base))
@@ -139,6 +146,13 @@ static int __init ls_pcie_ep_probe(struct
platform_device *pdev)
quoted
 	pci->ops = &ls_pcie_ep_ops;
 	pcie->pci = pci;

+	ls_epc->linkup_notifier = false,
+	ls_epc->msi_capable = true,
+	ls_epc->msix_capable = true,
As [msi,msix]_capable is shortly set from ls_pcie_ep_init - is there
any reason to set them here (to potentially incorrect values)?
This is a INIT value, maybe false is better for msi_capable and
msix_capable, of course, we don't need to set it.
ls_epc is kzalloc'd and so all zeros, so you get false for free. I think you can
remove these two lines (or all three if you don't care that linkup_notifier isn't
explicitly set).
Agree, This is correct, thanks a lot.
Thanks,

Andrew Murray
quoted
quoted
Thanks,

Andrew Murray
quoted
+	ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4),
+
+	pcie->ls_epc = ls_epc;
+
 	platform_set_drvdata(pdev, pcie);

 	ret = ls_add_pcie_ep(pcie, pdev);
--
2.9.5

RE: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of MSI-X in EP mode

From: Xiaowei Bao <hidden>
Date: 2019-08-16 11:15:01

-----Original Message-----
From: Kishon Vijay Abraham I <redacted>
Sent: 2019年8月16日 18:50
To: Xiaowei Bao <redacted>; Andrew Murray
[off-list ref]
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref];
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org; Z.q. Hou [off-list ref]
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode of
MSI-X in EP mode

Hi,

On 16/08/19 8:28 AM, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:54
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de; gregkh@linuxfoundation.org;
M.h. Lian [off-list ref]; Mingkai Hu
[off-list ref];
quoted
quoted
Roy Zang [off-list ref]; linux-pci@vger.kernel.org;
devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 02/10] PCI: designware-ep: Add the doorbell mode
of MSI-X in EP mode

On Thu, Aug 15, 2019 at 04:37:08PM +0800, Xiaowei Bao wrote:
quoted
Add the doorbell mode of MSI-X in EP mode.

Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c | 14
++++++++++++++
quoted
quoted
quoted
 drivers/pci/controller/dwc/pcie-designware.h    | 14
++++++++++++++
quoted
quoted
quoted
 2 files changed, 28 insertions(+)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 75e2955..e3a7cdf 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -454,6 +454,20 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep
quoted
quoted
*ep, u8 func_no,
quoted
 	return 0;
 }

+int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8
func_no,
quoted
+				       u16 interrupt_num)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 msg_data;
+
+	msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) |
+		   (interrupt_num - 1);
+
+	dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data);
+
+	return 0;
+}
+
 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
 			      u16 interrupt_num)
Have I understood correctly that the hardware provides an alternative
mechanism that allows for raising MSI-X interrupts without the bother
of reading the capabilities registers?
Yes, the hardware provide two way to MSI-X, please check the page 492
of
DWC_pcie_dm_registers_4.30 Menu.
MSIX_DOORBELL_OFF on page 492 0x948 Description: MSI-X Doorbell
Register....>
quoted
If so is there any good reason to keep dw_pcie_ep_raise_msix_irq?
(And thus use it in dw_plat_pcie_ep_raise_irq also)?
I am not sure, but I think the dw_pcie_ep_raise_msix_irq function is
not correct, because I think we can't get the MSIX table from the
address ep->phys_base + tbl_addr, but I also don't know where I can get the
correct MSIX table.

Sometime back when I tried raising MSI-X from EP, it was failing. It's quite
possible dw_pcie_ep_raise_msix_irq function is not correct.

MSI-X table can be obtained from the inbound ATU corresponding to the MSIX
bar.
IMO MSI-X support in EP mode needs rework. For instance set_msix should
also take BAR number as input to be configured in the MSI-X capability. The
function driver (pci-epf-test.c) should allocate memory taking into account the
MSI-X table.
Hi Kishon,

Thanks a lot for your explain, yes, we can get the MSI-X table from the inbound ATU of
the MSIX BAR.
Thanks
Kishon

Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Andrew Murray <hidden>
Date: 2019-08-16 12:35:32

On Fri, Aug 16, 2019 at 11:00:01AM +0000, Xiaowei Bao wrote:
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月16日 17:45
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
mark.rutland@arm.com; shawnguo@kernel.org; Leo Li
[off-list ref]; kishon@ti.com; lorenzo.pieralisi@arm.com;
arnd@arndb.de; gregkh@linuxfoundation.org; M.h. Lian
[off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org; Z.q. Hou [off-list ref]
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for
DWC

On Fri, Aug 16, 2019 at 02:55:41AM +0000, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:32
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de;
gregkh@linuxfoundation.org; M.h. Lian [off-list ref];
Mingkai Hu [off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs
support for DWC

On Thu, Aug 15, 2019 at 04:37:07PM +0800, Xiaowei Bao wrote:
quoted
Add multiple PFs support for DWC, different PF have different
config space, we use pf-offset property which get from the DTS to
access the different pF config space.
Thanks for the patch. I haven't seen a cover letter for this series,
is there one missing?
Maybe I miss, I will add you to review next time, thanks a lot for your
comments.
quoted
quoted
quoted
Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97
+++++++++++++---------
quoted
 drivers/pci/controller/dwc/pcie-designware.c    | 105
++++++++++++++++++++++--
quoted
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep
*ep)
quoted
quoted
quoted
 	pci_epc_linkup(epc);
 }

-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno
bar,
quoted
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no,
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
I think I'd rather see this arithmetic (and the one for determining
pf_base) inside a macro or inline header function. This would make
this code more readable and reduce the chances of an error by avoiding
duplication of code.
quoted
quoted
For example look at cdns_pcie_ep_fn_writeb and
ROCKCHIP_PCIE_EP_FUNC_BASE for examples of other EP drivers that do
this.
Agree, this looks fine, thanks a lot for your comments, I will use
this way to access the registers in next version patch.
quoted
quoted
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0); @@ -37,7 +39,12 @@ static
void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno
bar,

 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar)
{
quoted
quoted
quoted
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }

 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8
cap_ptr, @@ -78,28 +85,29 @@ static int
dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;

 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID,
hdr->vendorid);
quoted
quoted
quoted
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID, hdr->deviceid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID, hdr->revid);
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG,
hdr->progif_code);
quoted
quoted
quoted
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_VENDOR_ID,
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id);
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID,
hdr->subsys_id);
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);

 	return 0;
 }

-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum
pci_barno bar,
quoted
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)  {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct
dw_pcie_ep *ep, enum pci_barno bar,
quoted
 		return -EINVAL;
 	}

-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar,
+cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n"); @@
-125,7
quoted
quoted
quoted
+133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep,
enum pci_barno bar,
quoted
 	return 0;
 }

-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep,
phys_addr_t phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)  {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win,
PCIE_ATU_TYPE_MEM,
quoted
+				     phys_addr, pci_addr, size);

 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr; @@ -154,7 +163,7
@@
quoted
quoted
static
quoted
void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];

-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);

 	dw_pcie_disable_atu(pci, atu_index,
DW_PCIE_REGION_INBOUND);
quoted
quoted
quoted
 	clear_bit(atu_index, ep->ib_window_map); @@ -170,14 +179,16
@@
quoted
quoted
quoted
static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;

 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;

-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr,
as_type);
quoted
quoted
quoted
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc
*epc, u8 func_no,
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr,
+size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct pci_epc
*epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct pci_epc
*epc, u8 func_no, u8 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -287,12
+300,13 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8
func_no)  {
quoted
quoted
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct
pci_epc *epc, u8 func_no, u16 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep
quoted
quoted
*ep, u8 func_no,
quoted
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 		return -EINVAL;

 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1.
*/
quoted
quoted
quoted
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size - 1);
@@
quoted
quoted
quoted
-439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep
*ep,
u8 func_no,  }

 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no,
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc;
@@ -447,16 +463,17 @@ int dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;

-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower &
PCI_BASE_ADDRESS_MEM_TYPE_MASK);
quoted
quoted
@@
quoted
-592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);

-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions",
&epc->max_functions);
quoted
quoted
quoted
 	if (ret < 0)
 		epc->max_functions = 1;

+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
Bad things will likely happen if max_functions > 1 and pf-offset isn't set.
I think the driver should bail in this situation. It would be very
easy for someone to misconfigure this.
Yes, you are right, but if the max-functions have defined in DTS,
require the pf-offset must define in DTS, I am not sure the correct
value of pf-offsetfor other platforms, so I think the max-functions and
pf-offset should not have the dependence.

Yes you're correct. I hadn't really thought about this beyond layerscape. It's
also possible that other hardware could support multiple PFs without relying
on an offset and perhaps employ some other mechanism to access different
functions. So whilst this property can be optional for the majority of dwc
controllers - it must be set and cannot be zero for layerscape.

Perhaps inside ls_pcie_ep_init, you can set max_functions to 1 if pf_offset is
0 and print a WARN to explain why? (Or ls_pcie_ep_init returns failure and
dw_pcie_ep_init checks it and bails).

The assumption is being made here that future dw controllers may also use
pf_offset (is this likely?) - otherwise why is this in pcie-designware-ep.c and
not pci-layerscape-ep.c and why is this value not just hard-coded for lp?
Thanks a lot for your detail comments, this give me a lot of help.
Yes, I agree your point, and I will seriously consider a best way to fix this potential issue.
Based on your experience, how do other platforms implement the multiple functions?
The DWC core difference the different PF by signal "client0_tlp_func_num[(PF_WD-1):0]"
I don't know, though looking at the kernel drivers suggests that the
existing EP controllers have a large address space which contains multiple
PFs. They are accessed via macros (ROCKCHIP_PCIE_EP_FUNC_BASE(fn), 
CDNS_PCIE_EP_FUNC_BASE(fn)). It would be possible, but probably not desirable
to have a smaller address space (window) and a register that selects which
function the window refers to. This is why I'm slight nervous of assuming
that a pf-offset will cover all future dw drivers - I may be wrong.
quoted hunk
quoted
quoted
even though I didn't define pf-offset when I defined max-functions,
the pf-offset is 0, the DWC ep driver can continue run the progress of
INIT but not return, of course, thus the PF1 will not work, I don't know which
way is better.
Hi Andrew,
quoted
quoted
quoted
quoted
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c
b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void dw_pcie_writel_ob_unroll(struct
dw_pcie *pci, u32 index, u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

+static void dw_pcie_prog_ep_outbound_atu_unroll(struct dw_pcie
+*pci, u8
func_no,
quoted
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_LOWER_BASE,
quoted
quoted
quoted
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_UPPER_BASE,
quoted
quoted
quoted
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_LOWER_TARGET,
quoted
quoted
quoted
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_UPPER_TARGET,
quoted
quoted
quoted
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
quoted
+				 type | PCIE_ATU_FUNC_NUM(func_no));
With the exception of this line, the rest of this function is
identical to dw_pcie_prog_outbound_atu_unroll.
Yes, I can integrate the same code, but I think we'd better use the
different outbound window set function between RC and EP, because the RC
don't need the func_num parameter.


quoted
quoted
quoted
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL2,
quoted
quoted
quoted
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++)
{
quoted
quoted
quoted
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci,
int
index,
quoted
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size) @@ -194,6
+231,51 @@ static
quoted
quoted
quoted
void
dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");  }

+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8
+func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no, index,
type,
quoted
quoted
quoted
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
The same here, this is identical to dw_pcie_prog_outbound_atu with
the exception of this line.

Is there a way you can avoid all of this duplicated code?
As above, I can integrate the same code, but I keep to think the
different outbound Window set function should be used between RC and EP.
Or, is it possible to keep and use the existing functions, but use them
differently, e.g:

@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                return -EINVAL;
        }

-       dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
-                                 phys_addr, pci_addr, size);
+       dw_pcie_prog_outbound_atu(pci, free_win,
PCIE_ATU_TYPE_MEM_FUNC(func_no),
+                                    phys_addr, pci_addr, size);

        set_bit(free_win, ep->ob_window_map);
        ep->outbound_addr[free_win] = phys_addr;


Supported with:

#define PCIE_ATU_TYPE_MEM               0x0
#define PCIE_ATU_TYPE_MEM_FUNC(func_no) (PCIE_ATU_TYPE_MEM |
PCIE_ATU_FUNC_NUM(func_no))


This is just a suggestion, but I'm keen to avoid code duplication.
Thanks, I have think of a way as follow:

This is a good way, but I think PCIE_ATU_TYPE_MEM_FUNC(func_no) will give
Someone confused meaning, because PCIE_ATU_TYPE_MEM indicate the type of TLP,
and the location in the bit[0:3] of register CR1, but the PCIE_ATU_FUNC_NUM is bit[20:24],
I have another way:
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                 return -EINVAL;
        }
		
		dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
                                phys_addr, pci_addr, size);
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR1);
+		dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val | PCIE_ATU_FUNC_NUM(func_no));
or 
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
+                                   int type, u64 cpu_addr, u64 pci_addr,
+                                   u32 size)
+{
+		dw_pcie_prog_outbound_atu(pci, index, type, cpu_addr, pci_addr, size);	
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR1);
+		dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val | PCIE_ATU_FUNC_NUM(func_no));
+}

Which do you think is better of these three ways?
Building upon your idea, how about:

 
 @@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
 dw_pcie_ep *ep, phys_addr_t phys_addr,
                 return -EINVAL;
         }
 
 -       dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
 -                                 phys_addr, pci_addr, size);
 +       dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win, PCIE_ATU_TYPE_MEM
 +                                    phys_addr, pci_addr, size);
 
         set_bit(free_win, ep->ob_window_map);
         ep->outbound_addr[free_win] = phys_addr;
 

 +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
 +                                   int type, u64 cpu_addr, u64 pci_addr,
 +                                   u32 size)
 +{
 +		__dw_pcie_prog_outbound_atu(pci, func_no, index, type, cpu_addr, pci_addr, size);	
 +}
 +
 +void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, int index,
 +                                   int type, u64 cpu_addr, u64 pci_addr,
 +                                   u32 size)
 +{
 +		__dw_pcie_prog_outbound_atu(pci, 0, index, type, cpu_addr, pci_addr, size);	
 +}

In other words dw_pcie_prog_outbound_atu is updated (and renamed) to always
take a func_no and for host controllers this is always set to zero. Or you
could follow the approach taken in the cadence drivers for their implementation
of cdns_pcie_set_outbound_region - this always takes a func_no and is used
by host controller and endpoint drivers (except they don't have the helper
wrapper functions above thus exposing fn=0 to host controllers).
quoted
quoted
quoted
Thanks,

Andrew Murray
quoted
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent config
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++)
{
quoted
quoted
quoted
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int
type,
quoted
quoted
quoted
 			       u64 cpu_addr, u64 pci_addr, u32 size)  { @@
-252,8
quoted
quoted
+334,8
quoted
@@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32
index,
u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int
index,
quoted
quoted
quoted
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci,
+u8
func_no,
quoted
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)  {
 	int type;
@@ -275,8 +357,10 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct
dw_pcie *pci, int index,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
type);
quoted
+	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
type |
quoted
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL2,
quoted
quoted
quoted
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
@@ -297,14 +381,15 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	return -EBUSY;
 }

-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type)
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
quoted
quoted
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;

 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index,
+bar,
 						       cpu_addr, as_type);

 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
PCIE_ATU_REGION_INBOUND |
quoted
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie
*pci, int index, int bar,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));

 	/*
 	 * Make sure ATU enable takes effect before any subsequent
config diff --git a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie
*pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index,
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar,
-			     u64 cpu_addr, enum dw_pcie_as_type as_type);
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8
+func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
quoted
quoted
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);  void
dw_pcie_setup(struct
quoted
quoted
quoted
dw_pcie *pci); diff --git a/include/linux/pci-epc.h
b/include/linux/pci-epc.h index f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
Also pf_offset is an implementation detail needed only by the driver to
calculate where the PF is - it doesn't seem right that we share this with the EP
controller framework (whereas max_functions is used as a bounds check for
func_no in the framework calls).

I'd suggest that pf_offset is moved to a dwc structure, perhaps dw_pcie_ep?
I add the variable to this struct is consider that all PF is belong to a PCIe controller,
and the pci_epc indicate a PCIe controller, so I add this variable to this struct, what
do you think about this? I am not sure whether I should add this variable to dw_pcie_ep.
The EPC framework won't use the pf_offset and doesn't need it. It abstracts
the complexity of writing to the config address space (and similar) through
the pci_epc_ops. I'd suggest that the EPC framework (and pci_epc struct) only
needs to contain what *it* needs. Especially given that not all ep drivers
have a pf_offset or similar.

I understand the logic that pci_epc represents a EP controller, but I think
you should consider that it actually represents a *generic* EP controller in
the context of a framework which solely serves the purpose of connecting
controllers with functions. Whereas the dw_pcie_ep represents a specific type
of controller (DW) - as the pf_offset is (so far) relating to only DW
controllers (and as confirmed by the DT mapping) then it makes sense to not
move pf_offset from the specialised specific controller to the generic
controller. (Or at least this is how I rationalise it, though the EPC
framework is something quite unfamiliar to me).

Thanks,

Andrew Murray
quoted
Thanks,

Andrew Murray
quoted
quoted
quoted
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP controller */
 	spinlock_t			lock;
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;data=0
2
quoted
quoted
quoted
%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C0e39168f6f144db6840308d721742040%7
quoted
quoted
C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C637014654998524452&amp;sd
quoted
quoted
ata=bP7eh
quoted
cjlGXCMVFE2b4f12Q6fGV7lQ%2F5i9qIi9FoPlbI%3D&amp;reserved=0

RE: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for DWC

From: Xiaowei Bao <hidden>
Date: 2019-08-16 15:12:03

-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月16日 20:35
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
mark.rutland@arm.com; shawnguo@kernel.org; Leo Li
[off-list ref]; kishon@ti.com; lorenzo.pieralisi@arm.com;
arnd@arndb.de; gregkh@linuxfoundation.org; M.h. Lian
[off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org; Z.q. Hou [off-list ref]
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs support for
DWC

On Fri, Aug 16, 2019 at 11:00:01AM +0000, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月16日 17:45
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
mark.rutland@arm.com; shawnguo@kernel.org; Leo Li
[off-list ref]; kishon@ti.com; lorenzo.pieralisi@arm.com;
arnd@arndb.de; gregkh@linuxfoundation.org; M.h. Lian
[off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org; Z.q. Hou [off-list ref]
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs
support for DWC

On Fri, Aug 16, 2019 at 02:55:41AM +0000, Xiaowei Bao wrote:
quoted
quoted
-----Original Message-----
From: Andrew Murray <redacted>
Sent: 2019年8月15日 19:32
To: Xiaowei Bao <redacted>
Cc: jingoohan1@gmail.com; gustavo.pimentel@synopsys.com;
bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
shawnguo@kernel.org; Leo Li [off-list ref]; kishon@ti.com;
lorenzo.pieralisi@arm.com; arnd@arndb.de;
gregkh@linuxfoundation.org; M.h. Lian [off-list ref];
Mingkai Hu [off-list ref]; Roy Zang [off-list ref];
linux-pci@vger.kernel.org; devicetree@vger.kernel.org;
linux-kernel@vger.kernel.org;
linux-arm-kernel@lists.infradead.org;
linuxppc-dev@lists.ozlabs.org
Subject: Re: [PATCH 01/10] PCI: designware-ep: Add multiple PFs
support for DWC

On Thu, Aug 15, 2019 at 04:37:07PM +0800, Xiaowei Bao wrote:
quoted
Add multiple PFs support for DWC, different PF have different
config space, we use pf-offset property which get from the DTS
to access the different pF config space.
Thanks for the patch. I haven't seen a cover letter for this
series, is there one missing?
Maybe I miss, I will add you to review next time, thanks a lot for
your
comments.
quoted
quoted
quoted
Signed-off-by: Xiaowei Bao <redacted>
---
 drivers/pci/controller/dwc/pcie-designware-ep.c |  97
+++++++++++++---------
quoted
 drivers/pci/controller/dwc/pcie-designware.c    | 105
++++++++++++++++++++++--
quoted
 drivers/pci/controller/dwc/pcie-designware.h    |  10 ++-
 include/linux/pci-epc.h                         |   1 +
 4 files changed, 164 insertions(+), 49 deletions(-)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c
b/drivers/pci/controller/dwc/pcie-designware-ep.c
index 2bf5a35..75e2955 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -19,12 +19,14 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep
*ep)
quoted
quoted
quoted
 	pci_epc_linkup(epc);
 }

-static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno
bar,
quoted
-				   int flags)
+static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8
func_no,
quoted
quoted
quoted
quoted
quoted
+				   enum pci_barno bar, int flags)
 {
 	u32 reg;
+	struct pci_epc *epc = pci->ep.epc;
+	u32 pf_base = func_no * epc->pf_offset;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	reg = pf_base + PCI_BASE_ADDRESS_0 + (4 * bar);
I think I'd rather see this arithmetic (and the one for
determining
pf_base) inside a macro or inline header function. This would
make this code more readable and reduce the chances of an error
by avoiding
duplication of code.
quoted
quoted
For example look at cdns_pcie_ep_fn_writeb and
ROCKCHIP_PCIE_EP_FUNC_BASE for examples of other EP drivers that
do this.
Agree, this looks fine, thanks a lot for your comments, I will use
this way to access the registers in next version patch.
quoted
quoted
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_writel_dbi2(pci, reg, 0x0);
 	dw_pcie_writel_dbi(pci, reg, 0x0); @@ -37,7 +39,12 @@ static
void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum
pci_barno bar,

 void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno
bar)
{
quoted
quoted
quoted
-	__dw_pcie_ep_reset_bar(pci, bar, 0);
+	u8 func_no, funcs;
+
+	funcs = pci->ep.epc->max_functions;
+
+	for (func_no = 0; func_no < funcs; func_no++)
+		__dw_pcie_ep_reset_bar(pci, func_no, bar, 0);
 }

 static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8
cap_ptr, @@ -78,28 +85,29 @@ static int
dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no,  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;

 	dw_pcie_dbi_ro_wr_en(pci);
-	dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid);
-	dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid);
-	dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid);
-	dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code);
-	dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_VENDOR_ID,
hdr->vendorid);
quoted
quoted
quoted
+	dw_pcie_writew_dbi(pci, pf_base + PCI_DEVICE_ID,
hdr->deviceid);
quoted
quoted
quoted
quoted
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_REVISION_ID,
hdr->revid);
quoted
quoted
quoted
quoted
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CLASS_PROG,
hdr->progif_code);
quoted
quoted
quoted
+	dw_pcie_writew_dbi(pci, pf_base + PCI_CLASS_DEVICE,
 			   hdr->subclass_code | hdr->baseclass_code << 8);
-	dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE,
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_CACHE_LINE_SIZE,
 			   hdr->cache_line_size);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID,
+	dw_pcie_writew_dbi(pci, pf_base +
PCI_SUBSYSTEM_VENDOR_ID,
quoted
quoted
quoted
quoted
quoted
 			   hdr->subsys_vendor_id);
-	dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID,
hdr->subsys_id);
quoted
quoted
quoted
quoted
quoted
-	dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN,
+	dw_pcie_writew_dbi(pci, pf_base + PCI_SUBSYSTEM_ID,
hdr->subsys_id);
quoted
+	dw_pcie_writeb_dbi(pci, pf_base + PCI_INTERRUPT_PIN,
 			   hdr->interrupt_pin);
 	dw_pcie_dbi_ro_wr_dis(pci);

 	return 0;
 }

-static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum
pci_barno bar,
quoted
-				  dma_addr_t cpu_addr,
+static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
+				  enum pci_barno bar, dma_addr_t cpu_addr,
 				  enum dw_pcie_as_type as_type)  {
 	int ret;
@@ -112,7 +120,7 @@ static int dw_pcie_ep_inbound_atu(struct
dw_pcie_ep *ep, enum pci_barno bar,
quoted
 		return -EINVAL;
 	}

-	ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr,
+	ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar,
+cpu_addr,
 				       as_type);
 	if (ret < 0) {
 		dev_err(pci->dev, "Failed to program IB window\n"); @@
-125,7
quoted
quoted
quoted
+133,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep
+*ep,
enum pci_barno bar,
quoted
 	return 0;
 }

-static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep,
phys_addr_t phys_addr,
+static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
+				   phys_addr_t phys_addr,
 				   u64 pci_addr, size_t size)  {
 	u32 free_win;
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_prog_outbound_atu(pci, free_win,
PCIE_ATU_TYPE_MEM,
quoted
quoted
quoted
quoted
quoted
-				  phys_addr, pci_addr, size);
+	dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win,
PCIE_ATU_TYPE_MEM,
quoted
+				     phys_addr, pci_addr, size);

 	set_bit(free_win, ep->ob_window_map);
 	ep->outbound_addr[free_win] = phys_addr; @@ -154,7 +163,7
@@
quoted
quoted
static
quoted
void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no,
 	enum pci_barno bar = epf_bar->barno;
 	u32 atu_index = ep->bar_to_atu[bar];

-	__dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags);
+	__dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags);

 	dw_pcie_disable_atu(pci, atu_index,
DW_PCIE_REGION_INBOUND);
quoted
quoted
quoted
 	clear_bit(atu_index, ep->ib_window_map); @@ -170,14
+179,16
quoted
quoted
@@
quoted
quoted
quoted
static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no,
 	size_t size = epf_bar->size;
 	int flags = epf_bar->flags;
 	enum dw_pcie_as_type as_type;
-	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
+	u32 pf_base = func_no * epc->pf_offset;
+	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + pf_base;

 	if (!(flags & PCI_BASE_ADDRESS_SPACE))
 		as_type = DW_PCIE_AS_MEM;
 	else
 		as_type = DW_PCIE_AS_IO;

-	ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr,
as_type);
quoted
quoted
quoted
+	ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
+				     epf_bar->phys_addr, as_type);
 	if (ret)
 		return ret;
@@ -235,7 +246,7 @@ static int dw_pcie_ep_map_addr(struct
pci_epc
*epc, u8 func_no,
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);

-	ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size);
+	ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr,
+size);
 	if (ret) {
 		dev_err(pci->dev, "Failed to enable address\n");
 		return ret;
@@ -248,12 +259,13 @@ static int dw_pcie_ep_get_msi(struct
pci_epc *epc, u8 func_no)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSI_FLAGS_ENABLE))
 		return -EINVAL;
@@ -267,12 +279,13 @@ static int dw_pcie_ep_set_msi(struct
pci_epc *epc, u8 func_no, u8 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msi_cap)
 		return -EINVAL;

-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSI_FLAGS_QMASK;
 	val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@
-287,12
quoted
quoted
quoted
quoted
quoted
+300,13 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc,
+u8
func_no)  {
quoted
quoted
quoted
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	if (!(val & PCI_MSIX_FLAGS_ENABLE))
 		return -EINVAL;
@@ -306,12 +320,13 @@ static int dw_pcie_ep_set_msix(struct
pci_epc *epc, u8 func_no, u16 interrupts)  {
 	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	u32 pf_base = func_no * epc->pf_offset;
 	u32 val, reg;

 	if (!ep->msix_cap)
 		return -EINVAL;

-	reg = ep->msix_cap + PCI_MSIX_FLAGS;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_FLAGS;
 	val = dw_pcie_readw_dbi(pci, reg);
 	val &= ~PCI_MSIX_FLAGS_QSIZE;
 	val |= interrupts;
@@ -400,6 +415,7 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep
quoted
quoted
*ep, u8 func_no,
quoted
 	unsigned int aligned_offset;
 	u16 msg_ctrl, msg_data;
 	u32 msg_addr_lower, msg_addr_upper, reg;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 msg_addr;
 	bool has_upper;
 	int ret;
@@ -408,19 +424,19 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 		return -EINVAL;

 	/* Raise MSI per the PCI Local Bus Specification Revision 3.0,
6.8.1.
quoted
quoted
*/
quoted
quoted
quoted
-	reg = ep->msi_cap + PCI_MSI_FLAGS;
+	reg = ep->msi_cap + pf_base + PCI_MSI_FLAGS;
 	msg_ctrl = dw_pcie_readw_dbi(pci, reg);
 	has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT);
-	reg = ep->msi_cap + PCI_MSI_ADDRESS_LO;
+	reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_LO;
 	msg_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	if (has_upper) {
-		reg = ep->msi_cap + PCI_MSI_ADDRESS_HI;
+		reg = ep->msi_cap + pf_base + PCI_MSI_ADDRESS_HI;
 		msg_addr_upper = dw_pcie_readl_dbi(pci, reg);
-		reg = ep->msi_cap + PCI_MSI_DATA_64;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_64;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	} else {
 		msg_addr_upper = 0;
-		reg = ep->msi_cap + PCI_MSI_DATA_32;
+		reg = ep->msi_cap + pf_base + PCI_MSI_DATA_32;
 		msg_data = dw_pcie_readw_dbi(pci, reg);
 	}
 	aligned_offset = msg_addr_lower & (epc->mem->page_size -
1);
quoted
quoted
@@
quoted
quoted
quoted
-439,7 +455,7 @@ int dw_pcie_ep_raise_msi_irq(struct
dw_pcie_ep *ep,
u8 func_no,  }

 int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8
func_no,
quoted
quoted
quoted
quoted
quoted
-			     u16 interrupt_num)
+			      u16 interrupt_num)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
 	struct pci_epc *epc = ep->epc; @@ -447,16 +463,17 @@ int
dw_pcie_ep_raise_msix_irq(struct
dw_pcie_ep *ep, u8 func_no,
quoted
 	u32 bar_addr_upper, bar_addr_lower;
 	u32 msg_addr_upper, msg_addr_lower;
 	u32 reg, msg_data, vec_ctrl;
+	u32 pf_base = func_no * epc->pf_offset;
 	u64 tbl_addr, msg_addr, reg_u64;
 	void __iomem *msix_tbl;
 	int ret;

-	reg = ep->msix_cap + PCI_MSIX_TABLE;
+	reg = ep->msix_cap + pf_base + PCI_MSIX_TABLE;
 	tbl_offset = dw_pcie_readl_dbi(pci, reg);
 	bir = (tbl_offset & PCI_MSIX_TABLE_BIR);
 	tbl_offset &= PCI_MSIX_TABLE_OFFSET;

-	reg = PCI_BASE_ADDRESS_0 + (4 * bir);
+	reg = PCI_BASE_ADDRESS_0 + pf_base + (4 * bir);
 	bar_addr_upper = 0;
 	bar_addr_lower = dw_pcie_readl_dbi(pci, reg);
 	reg_u64 = (bar_addr_lower &
PCI_BASE_ADDRESS_MEM_TYPE_MASK);
quoted
quoted
@@
quoted
-592,13 +609,17 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep)
 	ep->epc = epc;
 	epc_set_drvdata(epc, ep);

-	if (ep->ops->ep_init)
-		ep->ops->ep_init(ep);
-
 	ret = of_property_read_u8(np, "max-functions",
&epc->max_functions);
quoted
quoted
quoted
 	if (ret < 0)
 		epc->max_functions = 1;

+	ret = of_property_read_u32(np, "pf-offset", &epc->pf_offset);
+	if (ret < 0)
+		epc->pf_offset = 0;
Bad things will likely happen if max_functions > 1 and pf-offset isn't
set.
quoted
quoted
quoted
quoted
I think the driver should bail in this situation. It would be
very easy for someone to misconfigure this.
Yes, you are right, but if the max-functions have defined in DTS,
require the pf-offset must define in DTS, I am not sure the
correct value of pf-offsetfor other platforms, so I think the
max-functions and
pf-offset should not have the dependence.

Yes you're correct. I hadn't really thought about this beyond
layerscape. It's also possible that other hardware could support
multiple PFs without relying on an offset and perhaps employ some
other mechanism to access different functions. So whilst this
property can be optional for the majority of dwc controllers - it must be
set and cannot be zero for layerscape.
quoted
quoted
Perhaps inside ls_pcie_ep_init, you can set max_functions to 1 if
pf_offset is
0 and print a WARN to explain why? (Or ls_pcie_ep_init returns
failure and dw_pcie_ep_init checks it and bails).

The assumption is being made here that future dw controllers may
also use pf_offset (is this likely?) - otherwise why is this in
pcie-designware-ep.c and not pci-layerscape-ep.c and why is this value
not just hard-coded for lp?
quoted
Thanks a lot for your detail comments, this give me a lot of help.
Yes, I agree your point, and I will seriously consider a best way to fix this
potential issue.
quoted
Based on your experience, how do other platforms implement the multiple
functions?
quoted
The DWC core difference the different PF by signal
"client0_tlp_func_num[(PF_WD-1):0]"

I don't know, though looking at the kernel drivers suggests that the existing EP
controllers have a large address space which contains multiple PFs. They are
accessed via macros (ROCKCHIP_PCIE_EP_FUNC_BASE(fn),
CDNS_PCIE_EP_FUNC_BASE(fn)). It would be possible, but probably not
desirable to have a smaller address space (window) and a register that selects
which function the window refers to. This is why I'm slight nervous of
assuming that a pf-offset will cover all future dw drivers - I may be wrong.
OK, thanks, maybe other people have good advice. I will use the macro to implement
the multiple function in v2 patch.
quoted
quoted
quoted
even though I didn't define pf-offset when I defined
max-functions, the pf-offset is 0, the DWC ep driver can continue
run the progress of INIT but not return, of course, thus the PF1
will not work, I don't know which
way is better.
Hi Andrew,
quoted
quoted
quoted
quoted
+
+	if (ep->ops->ep_init)
+		ep->ops->ep_init(ep);
+
 	ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size,
 				 ep->page_size);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/dwc/pcie-designware.c
b/drivers/pci/controller/dwc/pcie-designware.c
index 7d25102..c99cee4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -158,6 +158,43 @@ static void
dw_pcie_writel_ob_unroll(struct
dw_pcie *pci, u32 index, u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

+static void dw_pcie_prog_ep_outbound_atu_unroll(struct
+dw_pcie *pci, u8
func_no,
quoted
+						int index, int type,
+						u64 cpu_addr, u64 pci_addr,
+						u32 size)
+{
+	u32 retries, val;
+
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_LOWER_BASE,
quoted
quoted
quoted
+				 lower_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_UPPER_BASE,
quoted
quoted
quoted
+				 upper_32_bits(cpu_addr));
+	dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_LIMIT,
+				 lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_LOWER_TARGET,
quoted
quoted
quoted
+				 lower_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_UPPER_TARGET,
quoted
quoted
quoted
+				 upper_32_bits(pci_addr));
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
quoted
+				 type | PCIE_ATU_FUNC_NUM(func_no));
With the exception of this line, the rest of this function is
identical to dw_pcie_prog_outbound_atu_unroll.
Yes, I can integrate the same code, but I think we'd better use
the different outbound window set function between RC and EP,
because the RC
don't need the func_num parameter.


quoted
quoted
quoted
+	dw_pcie_writel_ob_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL2,
quoted
quoted
quoted
+				 PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent
config
quoted
quoted
quoted
quoted
quoted
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES;
+retries++)
{
quoted
quoted
quoted
+		val = dw_pcie_readl_ob_unroll(pci, index,
+					      PCIE_ATU_UNR_REGION_CTRL2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie
*pci, int
index,
quoted
 					     int type, u64 cpu_addr,
 					     u64 pci_addr, u32 size) @@ -194,6
+231,51 @@ static
quoted
quoted
quoted
void
dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	dev_err(pci->dev, "Outbound iATU is not being enabled\n");
}

+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8
+func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size)
+{
+	u32 retries, val;
+
+	if (pci->ops->cpu_addr_fixup)
+		cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr);
+
+	if (pci->iatu_unroll_enabled) {
+		dw_pcie_prog_ep_outbound_atu_unroll(pci, func_no,
index,
quoted
quoted
type,
quoted
quoted
quoted
+						    cpu_addr, pci_addr, size);
+		return;
+	}
+
+	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
+			   PCIE_ATU_REGION_OUTBOUND | index);
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_BASE,
+			   lower_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_BASE,
+			   upper_32_bits(cpu_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LIMIT,
+			   lower_32_bits(cpu_addr + size - 1));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_LOWER_TARGET,
+			   lower_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET,
+			   upper_32_bits(pci_addr));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
The same here, this is identical to dw_pcie_prog_outbound_atu
with the exception of this line.

Is there a way you can avoid all of this duplicated code?
As above, I can integrate the same code, but I keep to think the
different outbound Window set function should be used between RC and
EP.
quoted
quoted
Or, is it possible to keep and use the existing functions, but use
them differently, e.g:

@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                return -EINVAL;
        }

-       dw_pcie_prog_outbound_atu(pci, free_win,
PCIE_ATU_TYPE_MEM,
quoted
quoted
-                                 phys_addr, pci_addr, size);
+       dw_pcie_prog_outbound_atu(pci, free_win,
PCIE_ATU_TYPE_MEM_FUNC(func_no),
+                                    phys_addr, pci_addr, size);

        set_bit(free_win, ep->ob_window_map);
        ep->outbound_addr[free_win] = phys_addr;


Supported with:

#define PCIE_ATU_TYPE_MEM               0x0
#define PCIE_ATU_TYPE_MEM_FUNC(func_no) (PCIE_ATU_TYPE_MEM |
PCIE_ATU_FUNC_NUM(func_no))


This is just a suggestion, but I'm keen to avoid code duplication.
Thanks, I have think of a way as follow:

This is a good way, but I think PCIE_ATU_TYPE_MEM_FUNC(func_no) will
give Someone confused meaning, because PCIE_ATU_TYPE_MEM indicate
the
quoted
type of TLP, and the location in the bit[0:3] of register CR1, but the
PCIE_ATU_FUNC_NUM is bit[20:24], I have another way:
@@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                 return -EINVAL;
        }

		dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM,
                                phys_addr, pci_addr, size);
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR1);
+		dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val |
+PCIE_ATU_FUNC_NUM(func_no));
or
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
quoted
+                                   int type, u64 cpu_addr, u64
pci_addr,
quoted
+                                   u32 size) {
+		dw_pcie_prog_outbound_atu(pci, index, type, cpu_addr, pci_addr,
size);
quoted
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR1);
+		dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, val |
+PCIE_ATU_FUNC_NUM(func_no)); }

Which do you think is better of these three ways?
Building upon your idea, how about:


 @@ -137,8 +146,8 @@ static int dw_pcie_ep_outbound_atu(struct
dw_pcie_ep *ep, phys_addr_t phys_addr,
                 return -EINVAL;
         }

 -       dw_pcie_prog_outbound_atu(pci, free_win,
PCIE_ATU_TYPE_MEM,
 -                                 phys_addr, pci_addr, size);
 +       dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win,
PCIE_ATU_TYPE_MEM
 +                                    phys_addr, pci_addr, size);

         set_bit(free_win, ep->ob_window_map);
         ep->outbound_addr[free_win] = phys_addr;


 +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
 +                                   int type, u64 cpu_addr, u64
pci_addr,
 +                                   u32 size)
 +{
 +		__dw_pcie_prog_outbound_atu(pci, func_no, index, type, cpu_addr,
pci_addr, size);
 +}
 +
 +void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, int
index,
 +                                   int type, u64 cpu_addr, u64
pci_addr,
 +                                   u32 size)
 +{
 +		__dw_pcie_prog_outbound_atu(pci, 0, index, type, cpu_addr,
pci_addr, size);
 +}

In other words dw_pcie_prog_outbound_atu is updated (and renamed) to
always take a func_no and for host controllers this is always set to zero. Or
you could follow the approach taken in the cadence drivers for their
implementation of cdns_pcie_set_outbound_region - this always takes a
func_no and is used by host controller and endpoint drivers (except they don't
have the helper wrapper functions above thus exposing fn=0 to host
controllers).
You're correct, I think this way is better, thanks.
quoted
quoted
quoted
quoted
Thanks,

Andrew Murray
quoted
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE);
+
+	/*
+	 * Make sure ATU enable takes effect before any subsequent
config
quoted
quoted
quoted
quoted
quoted
+	 * and I/O accesses.
+	 */
+	for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES;
+retries++)
{
quoted
quoted
quoted
+		val = dw_pcie_readl_dbi(pci, PCIE_ATU_CR2);
+		if (val & PCIE_ATU_ENABLE)
+			return;
+
+		mdelay(LINK_WAIT_IATU);
+	}
+	dev_err(pci->dev, "Outbound iATU is not being enabled\n"); }
+
 void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int
index, int
type,
quoted
quoted
quoted
 			       u64 cpu_addr, u64 pci_addr, u32 size)
{ @@
quoted
quoted
-252,8
quoted
quoted
+334,8
quoted
@@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci,
u32 index,
u32 reg,
quoted
 	dw_pcie_writel_atu(pci, offset + reg, val);  }

-static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie
*pci, int
index,
quoted
quoted
quoted
-					   int bar, u64 cpu_addr,
+static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie
+*pci,
+u8
func_no,
quoted
+					   int index, int bar, u64 cpu_addr,
 					   enum dw_pcie_as_type as_type)  {
 	int type;
@@ -275,8 +357,10 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct
dw_pcie *pci, int index,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
type);
quoted
+	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL1,
quoted
quoted
type |
quoted
+				 PCIE_ATU_FUNC_NUM(func_no));
 	dw_pcie_writel_ib_unroll(pci, index,
PCIE_ATU_UNR_REGION_CTRL2,
quoted
quoted
quoted
+				 PCIE_ATU_FUNC_NUM_MATCH_EN |
 				 PCIE_ATU_ENABLE |
 				 PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
@@ -297,14 +381,15 @@ static int
dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index,
quoted
 	return -EBUSY;
 }

-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int
bar,
quoted
quoted
quoted
quoted
quoted
-			     u64 cpu_addr, enum dw_pcie_as_type
as_type)
quoted
quoted
quoted
quoted
quoted
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no,
+int
index,
quoted
quoted
quoted
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type)
 {
 	int type;
 	u32 retries, val;

 	if (pci->iatu_unroll_enabled)
-		return dw_pcie_prog_inbound_atu_unroll(pci, index, bar,
+		return dw_pcie_prog_inbound_atu_unroll(pci, func_no,
index,
quoted
quoted
quoted
quoted
quoted
+bar,
 						       cpu_addr, as_type);

 	dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT,
PCIE_ATU_REGION_INBOUND |
quoted
@@ -323,9 +408,11 @@ int dw_pcie_prog_inbound_atu(struct
dw_pcie
*pci, int index, int bar,
quoted
 		return -EINVAL;
 	}

-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type);
-	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE
-			   | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type |
+			   PCIE_ATU_FUNC_NUM(func_no));
+	dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE |
+			   PCIE_ATU_FUNC_NUM_MATCH_EN |
+			   PCIE_ATU_BAR_MODE_ENABLE | (bar << 8));

 	/*
 	 * Make sure ATU enable takes effect before any subsequent
config diff --git
a/drivers/pci/controller/dwc/pcie-designware.h
b/drivers/pci/controller/dwc/pcie-designware.h
index ffed084..2b291e8 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -71,9 +71,11 @@
 #define PCIE_ATU_TYPE_IO		0x2
 #define PCIE_ATU_TYPE_CFG0		0x4
 #define PCIE_ATU_TYPE_CFG1		0x5
+#define PCIE_ATU_FUNC_NUM(pf)           (pf << 20)
 #define PCIE_ATU_CR2			0x908
 #define PCIE_ATU_ENABLE			BIT(31)
 #define PCIE_ATU_BAR_MODE_ENABLE	BIT(30)
+#define PCIE_ATU_FUNC_NUM_MATCH_EN      BIT(19)
 #define PCIE_ATU_LOWER_BASE		0x90C
 #define PCIE_ATU_UPPER_BASE		0x910
 #define PCIE_ATU_LIMIT			0x914
@@ -265,8 +267,12 @@ int dw_pcie_wait_for_link(struct dw_pcie
*pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int
index,
quoted
quoted
quoted
quoted
quoted
 			       int type, u64 cpu_addr, u64 pci_addr,
 			       u32 size);
-int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int
bar,
quoted
quoted
quoted
quoted
quoted
-			     u64 cpu_addr, enum dw_pcie_as_type
as_type);
quoted
quoted
quoted
quoted
quoted
+void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8
+func_no, int
index,
quoted
+				  int type, u64 cpu_addr, u64 pci_addr,
+				  u32 size);
+int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no,
+int
index,
quoted
quoted
quoted
+			     int bar, u64 cpu_addr,
+			     enum dw_pcie_as_type as_type);
 void dw_pcie_disable_atu(struct dw_pcie *pci, int index,
 			 enum dw_pcie_region_type type);  void
dw_pcie_setup(struct
quoted
quoted
quoted
dw_pcie *pci); diff --git a/include/linux/pci-epc.h
b/include/linux/pci-epc.h index f641bad..fc2feee 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -96,6 +96,7 @@ struct pci_epc {
 	const struct pci_epc_ops	*ops;
 	struct pci_epc_mem		*mem;
 	u8				max_functions;
+	u32				pf_offset;
Also pf_offset is an implementation detail needed only by the driver
to calculate where the PF is - it doesn't seem right that we share
this with the EP controller framework (whereas max_functions is used
as a bounds check for func_no in the framework calls).

I'd suggest that pf_offset is moved to a dwc structure, perhaps
dw_pcie_ep?
quoted
I add the variable to this struct is consider that all PF is belong to
a PCIe controller, and the pci_epc indicate a PCIe controller, so I
add this variable to this struct, what do you think about this? I am not sure
whether I should add this variable to dw_pcie_ep.

The EPC framework won't use the pf_offset and doesn't need it. It abstracts
the complexity of writing to the config address space (and similar) through the
pci_epc_ops. I'd suggest that the EPC framework (and pci_epc struct) only
needs to contain what *it* needs. Especially given that not all ep drivers have
a pf_offset or similar.

I understand the logic that pci_epc represents a EP controller, but I think you
should consider that it actually represents a *generic* EP controller in the
context of a framework which solely serves the purpose of connecting
controllers with functions. Whereas the dw_pcie_ep represents a specific
type of controller (DW) - as the pf_offset is (so far) relating to only DW
controllers (and as confirmed by the DT mapping) then it makes sense to not
move pf_offset from the specialised specific controller to the generic
controller. (Or at least this is how I rationalise it, though the EPC framework is
something quite unfamiliar to me).
I think this is more reasonable by your detail explaining, I will move pf_offset
to the struct dw_pcie_ep, thanks again!
Thanks,

Andrew Murray
quoted
quoted
Thanks,

Andrew Murray
quoted
quoted
quoted
 	struct config_group		*group;
 	/* spinlock to protect against concurrent access of EP
controller */
quoted
quoted
quoted
quoted
quoted
 	spinlock_t			lock;
--
2.9.5


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists
.infradead.org%2Fmailman%2Flistinfo%2Flinux-arm-kernel&amp;dat
a=0
2
quoted
quoted
quoted
%
7C0
quoted
1%7Cxiaowei.bao%40nxp.com%7C0e39168f6f144db6840308d721742040%7
quoted
quoted
quoted
quoted
C686ea1d
quoted
3bc2b4c6fa92cd99c5c301635%7C0%7C1%7C637014654998524452&amp;sd
quoted
quoted
quoted
quoted
ata=bP7eh
quoted
cjlGXCMVFE2b4f12Q6fGV7lQ%2F5i9qIi9FoPlbI%3D&amp;reserved=0

Re: [PATCH 08/10] dt-bindings: PCI: Add the pf-offset property

From: Rob Herring <robh@kernel.org>
Date: 2019-08-27 16:26:52

On Thu, 15 Aug 2019 16:37:14 +0800, Xiaowei Bao wrote:
Add the pf-offset property for multiple PF.

Signed-off-by: Xiaowei Bao <redacted>
---
 Documentation/devicetree/bindings/pci/designware-pcie.txt | 1 +
 1 file changed, 1 insertion(+)
Reviewed-by: Rob Herring <robh@kernel.org>
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help