[PATCH] cxl: Add support for POWER9 DD2

Subsystems: char and misc drivers, the rest

STALE3299d

5 messages, 3 authors, 2017-08-24 · open the first message on its own page

[PATCH] cxl: Add support for POWER9 DD2

From: Christophe Lombard <hidden>
Date: 2017-08-23 14:58:17

The PSL initialization sequence has been updated to DD2.
This patch adapts to the changes, retaining compatibility with DD1.

Tests performed on some of the new hardware.

Signed-off-by: Christophe Lombard <redacted>
---
 drivers/misc/cxl/cxl.h |  2 ++
 drivers/misc/cxl/pci.c | 57 +++++++++++++++++++++++++++++++-------------------
 2 files changed, 38 insertions(+), 21 deletions(-)
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index b1afecc..0167df8 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -100,6 +100,8 @@ static const cxl_p1_reg_t CXL_XSL_FEC       = {0x0158};
 static const cxl_p1_reg_t CXL_XSL_DSNCTL    = {0x0168};
 /* PSL registers - CAIA 2 */
 static const cxl_p1_reg_t CXL_PSL9_CONTROL  = {0x0020};
+static const cxl_p1_reg_t CXL_XSL9_INV      = {0x0110};
+static const cxl_p1_reg_t CXL_XSL9_DEF      = {0x0140};
 static const cxl_p1_reg_t CXL_XSL9_DSNCTL   = {0x0168};
 static const cxl_p1_reg_t CXL_PSL9_FIR1     = {0x0300};
 static const cxl_p1_reg_t CXL_PSL9_FIR2     = {0x0308};
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index d18b3d9..a981c65 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -475,37 +475,52 @@ static int init_implementation_adapter_regs_psl9(struct cxl *adapter,
 	psl_fircntl |= 0x1ULL; /* ce_thresh */
 	cxl_p1_write(adapter, CXL_PSL9_FIR_CNTL, psl_fircntl);
 
-	/* vccredits=0x1  pcklat=0x4 */
-	cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0000000000001810ULL);
-
-	/*
-	 * For debugging with trace arrays.
-	 * Configure RX trace 0 segmented mode.
-	 * Configure CT trace 0 segmented mode.
-	 * Configure LA0 trace 0 segmented mode.
-	 * Configure LA1 trace 0 segmented mode.
+	/* Setup the PSL to transmit packets on the PCIe before the
+	 * CAPP is enabled
 	 */
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000000ULL);
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000003ULL);
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000005ULL);
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000006ULL);
+	cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0001001000002A10ULL);
+
+	/* For debugging with trace arrays */
+	/* Configure RX trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000000ULL);
+	/* Configure RX trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xAA00000000000001ULL);
+	/* Configure CT trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xA2B8000000000003ULL);
+	/* Configure LA0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x83FFC00000000005ULL);
+	/* Configure JM0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000007ULL);
+	/* Configure DMA trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000008ULL);
+	/* Configure DMA trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000009ULL);
 
 	/*
 	 * A response to an ASB_Notify request is returned by the
 	 * system as an MMIO write to the address defined in
 	 * the PSL_TNR_ADDR register
 	 */
-	/* PSL_TNR_ADDR */
+	/* keep the Reset Value: 0x00020000E0000000 */
+
+	/* Enable XSL rty limit */
+	cxl_p1_write(adapter, CXL_XSL9_DEF, 0x51F8000000000005ULL);
 
-	/* NORST */
-	cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0x8000000000000000ULL);
+	/* Change XSL_INV dummy readtheshold */
+	cxl_p1_write(adapter, CXL_XSL9_INV, 0x0000040007FFC200ULL);
 
-	/* allocate the apc machines */
-	cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 0x40000003FFFF0000ULL);
+	if (phb_index == 3) {
+		/* disable machines 31-47 and 20-27 for DMA */
+		cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 0x40000FF3FFFF0000ULL);
+	}
+
+	/* Snoop machines */
+	cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL);
 
-	/* Disable vc dd1 fix */
-	if (cxl_is_power9_dd1())
-		cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0400000000000001ULL);
+	if (cxl_is_power9_dd1()) {
+		/* Disabling deadlock counter CAR */
+		cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0020000000000001ULL);
+	}
 
 	return 0;
 }
-- 
2.7.4

Re: [PATCH] cxl: Add support for POWER9 DD2

From: Andrew Donnellan <hidden>
Date: 2017-08-24 05:24:53

On 24/08/17 00:58, Christophe Lombard wrote:
The PSL initialization sequence has been updated to DD2.
This patch adapts to the changes, retaining compatibility with DD1.

Tests performed on some of the new hardware.
If we're retaining compatibility with DD1 I assume it's been tested on 
some of the old hardware too?

It seems this includes some changes to DD1 fix-ups as well.
quoted hunk
Signed-off-by: Christophe Lombard <redacted> > ---
  drivers/misc/cxl/cxl.h |  2 ++
  drivers/misc/cxl/pci.c | 57 +++++++++++++++++++++++++++++++-------------------
  2 files changed, 38 insertions(+), 21 deletions(-)
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index b1afecc..0167df8 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -100,6 +100,8 @@ static const cxl_p1_reg_t CXL_XSL_FEC       = {0x0158};
  static const cxl_p1_reg_t CXL_XSL_DSNCTL    = {0x0168};
  /* PSL registers - CAIA 2 */
  static const cxl_p1_reg_t CXL_PSL9_CONTROL  = {0x0020};
+static const cxl_p1_reg_t CXL_XSL9_INV      = {0x0110};
+static const cxl_p1_reg_t CXL_XSL9_DEF      = {0x0140};
  static const cxl_p1_reg_t CXL_XSL9_DSNCTL   = {0x0168};
  static const cxl_p1_reg_t CXL_PSL9_FIR1     = {0x0300};
  static const cxl_p1_reg_t CXL_PSL9_FIR2     = {0x0308};
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index d18b3d9..a981c65 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -475,37 +475,52 @@ static int init_implementation_adapter_regs_psl9(struct cxl *adapter,
  	psl_fircntl |= 0x1ULL; /* ce_thresh */
  	cxl_p1_write(adapter, CXL_PSL9_FIR_CNTL, psl_fircntl);
  
-	/* vccredits=0x1  pcklat=0x4 */
-	cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0000000000001810ULL);
-
-	/*
-	 * For debugging with trace arrays.
-	 * Configure RX trace 0 segmented mode.
-	 * Configure CT trace 0 segmented mode.
-	 * Configure LA0 trace 0 segmented mode.
-	 * Configure LA1 trace 0 segmented mode.
+	/* Setup the PSL to transmit packets on the PCIe before the
+	 * CAPP is enabled
  	 */
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000000ULL);
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000003ULL);
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000005ULL);
-	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000006ULL);
+	cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0001001000002A10ULL);
+
+	/* For debugging with trace arrays */
+	/* Configure RX trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000000ULL);
+	/* Configure RX trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xAA00000000000001ULL);
+	/* Configure CT trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xA2B8000000000003ULL);
+	/* Configure LA0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x83FFC00000000005ULL);
+	/* Configure JM0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000007ULL);
+	/* Configure DMA trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000008ULL);
+	/* Configure DMA trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000009ULL);
  
  	/*
  	 * A response to an ASB_Notify request is returned by the
  	 * system as an MMIO write to the address defined in
  	 * the PSL_TNR_ADDR register
  	 */
-	/* PSL_TNR_ADDR */
+	/* keep the Reset Value: 0x00020000E0000000 */
I was confused by this comment for a while - maybe keep PSL_TNR_ADDR at 
the beginning of the comment, it's not completely clear from the 
previous block alone.
+
+	/* Enable XSL rty limit */
+	cxl_p1_write(adapter, CXL_XSL9_DEF, 0x51F8000000000005ULL);
  
-	/* NORST */
-	cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0x8000000000000000ULL);
+	/* Change XSL_INV dummy readtheshold */
read threshold?
+	cxl_p1_write(adapter, CXL_XSL9_INV, 0x0000040007FFC200ULL);
  
-	/* allocate the apc machines */
-	cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 0x40000003FFFF0000ULL);
+	if (phb_index == 3) {
+		/* disable machines 31-47 and 20-27 for DMA */
+		cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 0x40000FF3FFFF0000ULL);
+	}
+
+	/* Snoop machines */
+	cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL);
  
-	/* Disable vc dd1 fix */
-	if (cxl_is_power9_dd1())
-		cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0400000000000001ULL);
+	if (cxl_is_power9_dd1()) {
+		/* Disabling deadlock counter CAR */
+		cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0020000000000001ULL);
+	}
  
  	return 0;
  }
-- 
Andrew Donnellan              OzLabs, ADL Canberra
andrew.donnellan@au1.ibm.com  IBM Australia Limited

Re: [PATCH] cxl: Add support for POWER9 DD2

From: Vaibhav Jain <hidden>
Date: 2017-08-24 07:09:40

Hi Christophe,

Christophe Lombard [off-list ref] writes:
+	/* For debugging with trace arrays */
+	/* Configure RX trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000000ULL);
+	/* Configure RX trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xAA00000000000001ULL);
+	/* Configure CT trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xA2B8000000000003ULL);
+	/* Configure LA0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x83FFC00000000005ULL);
+	/* Configure JM0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000007ULL);
+	/* Configure DMA trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000008ULL);
+	/* Configure DMA trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000009ULL);
Please wrap this block that configures the trace arrays in #ifdef
DEBUG. Or it will be better if we remove it from here as the register is
already accessible via debugfs.

---
Vaibhav Jain [off-list ref]
Linux Technology Center, IBM India Pvt. Ltd.

Re: [PATCH] cxl: Add support for POWER9 DD2

From: christophe lombard <hidden>
Date: 2017-08-24 07:20:53

Le 24/08/2017 à 09:09, Vaibhav Jain a écrit :
Hi Christophe,

Christophe Lombard [off-list ref] writes:
quoted
+	/* For debugging with trace arrays */
+	/* Configure RX trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000000ULL);
+	/* Configure RX trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xAA00000000000001ULL);
+	/* Configure CT trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xA2B8000000000003ULL);
+	/* Configure LA0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x83FFC00000000005ULL);
+	/* Configure JM0 trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000007ULL);
+	/* Configure DMA trace 0 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000008ULL);
+	/* Configure DMA trace 1 segmented mode */
+	cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000009ULL);
Please wrap this block that configures the trace arrays in #ifdef
DEBUG. Or it will be better if we remove it from here as the register is
already accessible via debugfs.

---
Vaibhav Jain [off-list ref]
Linux Technology Center, IBM India Pvt. Ltd.
okay, I will remove this block

Thanks

Re: [PATCH] cxl: Add support for POWER9 DD2

From: christophe lombard <hidden>
Date: 2017-08-24 07:23:14

Le 24/08/2017 à 07:24, Andrew Donnellan a écrit :
On 24/08/17 00:58, Christophe Lombard wrote:
quoted
The PSL initialization sequence has been updated to DD2.
This patch adapts to the changes, retaining compatibility with DD1.

Tests performed on some of the new hardware.
If we're retaining compatibility with DD1 I assume it's been tested on 
some of the old hardware too?
right, it's been tested on boston machine with dd1
It seems this includes some changes to DD1 fix-ups as well.
correct
quoted
Signed-off-by: Christophe Lombard <redacted> > ---
  drivers/misc/cxl/cxl.h |  2 ++
  drivers/misc/cxl/pci.c | 57 
+++++++++++++++++++++++++++++++-------------------
  2 files changed, 38 insertions(+), 21 deletions(-)
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index b1afecc..0167df8 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -100,6 +100,8 @@ static const cxl_p1_reg_t CXL_XSL_FEC = {0x0158};
  static const cxl_p1_reg_t CXL_XSL_DSNCTL    = {0x0168};
  /* PSL registers - CAIA 2 */
  static const cxl_p1_reg_t CXL_PSL9_CONTROL  = {0x0020};
+static const cxl_p1_reg_t CXL_XSL9_INV      = {0x0110};
+static const cxl_p1_reg_t CXL_XSL9_DEF      = {0x0140};
  static const cxl_p1_reg_t CXL_XSL9_DSNCTL   = {0x0168};
  static const cxl_p1_reg_t CXL_PSL9_FIR1     = {0x0300};
  static const cxl_p1_reg_t CXL_PSL9_FIR2     = {0x0308};
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index d18b3d9..a981c65 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -475,37 +475,52 @@ static int 
init_implementation_adapter_regs_psl9(struct cxl *adapter,
      psl_fircntl |= 0x1ULL; /* ce_thresh */
      cxl_p1_write(adapter, CXL_PSL9_FIR_CNTL, psl_fircntl);
  -    /* vccredits=0x1  pcklat=0x4 */
-    cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0000000000001810ULL);
-
-    /*
-     * For debugging with trace arrays.
-     * Configure RX trace 0 segmented mode.
-     * Configure CT trace 0 segmented mode.
-     * Configure LA0 trace 0 segmented mode.
-     * Configure LA1 trace 0 segmented mode.
+    /* Setup the PSL to transmit packets on the PCIe before the
+     * CAPP is enabled
       */
-    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000000ULL);
-    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000003ULL);
-    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000005ULL);
-    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8040800080000006ULL);
+    cxl_p1_write(adapter, CXL_PSL9_DSNDCTL, 0x0001001000002A10ULL);
+
+    /* For debugging with trace arrays */
+    /* Configure RX trace 0 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000000ULL);
+    /* Configure RX trace 1 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xAA00000000000001ULL);
+    /* Configure CT trace 0 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0xA2B8000000000003ULL);
+    /* Configure LA0 trace 0 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x83FFC00000000005ULL);
+    /* Configure JM0 trace 0 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000007ULL);
+    /* Configure DMA trace 0 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000008ULL);
+    /* Configure DMA trace 1 segmented mode */
+    cxl_p1_write(adapter, CXL_PSL9_TRACECFG, 0x8200000000000009ULL);
        /*
       * A response to an ASB_Notify request is returned by the
       * system as an MMIO write to the address defined in
       * the PSL_TNR_ADDR register
       */
-    /* PSL_TNR_ADDR */
+    /* keep the Reset Value: 0x00020000E0000000 */
I was confused by this comment for a while - maybe keep PSL_TNR_ADDR 
at the beginning of the comment, it's not completely clear from the 
previous block alone.
okay, will do.
quoted
+
+    /* Enable XSL rty limit */
+    cxl_p1_write(adapter, CXL_XSL9_DEF, 0x51F8000000000005ULL);
  -    /* NORST */
-    cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0x8000000000000000ULL);
+    /* Change XSL_INV dummy readtheshold */
read threshold?
quoted
+    cxl_p1_write(adapter, CXL_XSL9_INV, 0x0000040007FFC200ULL);
  -    /* allocate the apc machines */
-    cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 0x40000003FFFF0000ULL);
+    if (phb_index == 3) {
+        /* disable machines 31-47 and 20-27 for DMA */
+        cxl_p1_write(adapter, CXL_PSL9_APCDEDTYPE, 
0x40000FF3FFFF0000ULL);
+    }
+
+    /* Snoop machines */
+    cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL);
  -    /* Disable vc dd1 fix */
-    if (cxl_is_power9_dd1())
-        cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0400000000000001ULL);
+    if (cxl_is_power9_dd1()) {
+        /* Disabling deadlock counter CAR */
+        cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0020000000000001ULL);
+    }
        return 0;
  }
Keyboard shortcuts
hback out one level
jnext message in thread
kprevious message in thread
ldrill in
Escclose help / fold thread tree
?toggle this help