[PATCH] powerpc/drmem: Don't compute the NUMA node for each LMB

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Signed-off-by: Laurent Dufour <redacted>
---
 arch/powerpc/mm/drmem.c | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..3819c523c65b 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c

@@ -397,7 +397,7 @@ static void __init init_drmem_v1_lmbs(const __be32 *prop)
 
 static void __init init_drmem_v2_lmbs(const __be32 *prop)
 {
-	struct drmem_lmb *lmb;
+	struct drmem_lmb *lmb, *first;
 	struct of_drconf_cell_v2 dr_cell;
 	const __be32 *p;
 	u32 i, j, lmb_sets;

@@ -422,10 +422,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	/* second pass, read in the LMB information */
 	lmb_index = 0;
 	p = prop;
+	first = NULL;
 
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 

@@ -438,7 +446,16 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
 		}
 	}
 }

-- 
2.28.0

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <redacted>

All errors (new ones prefixed by >>):

   arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':

quoted

arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'

     457 |     lmb->nid = first->nid;
         |        ^~
   arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
     457 |     lmb->nid = first->nid;
         |                     ^~

vim +457 arch/powerpc/mm/drmem.c

   397	
   398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
   399	{
   400		struct drmem_lmb *lmb, *first;
   401		struct of_drconf_cell_v2 dr_cell;
   402		const __be32 *p;
   403		u32 i, j, lmb_sets;
   404		int lmb_index;
   405	
   406		lmb_sets = of_read_number(prop++, 1);
   407		if (lmb_sets == 0)
   408			return;
   409	
   410		/* first pass, calculate the number of LMBs */
   411		p = prop;
   412		for (i = 0; i < lmb_sets; i++) {
   413			read_drconf_v2_cell(&dr_cell, &p);
   414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
   415		}
   416	
   417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
   418					   GFP_KERNEL);
   419		if (!drmem_info->lmbs)
   420			return;
   421	
   422		/* second pass, read in the LMB information */
   423		lmb_index = 0;
   424		p = prop;
   425		first = NULL;
   426	
   427		for (i = 0; i < lmb_sets; i++) {
   428			read_drconf_v2_cell(&dr_cell, &p);
   429	
   430			/*
   431			 * Fetch the NUMA node id for the fist set or if the
   432			 * associativity index is different from the previous set.
   433			 */
   434			if (first && dr_cell.aa_index != first->aa_index)
   435				first = NULL;
   436	
   437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
   438				lmb = &drmem_info->lmbs[lmb_index++];
   439	
   440				lmb->base_addr = dr_cell.base_addr;
   441				dr_cell.base_addr += drmem_info->lmb_size;
   442	
   443				lmb->drc_index = dr_cell.drc_index;
   444				dr_cell.drc_index++;
   445	
   446				lmb->aa_index = dr_cell.aa_index;
   447				lmb->flags = dr_cell.flags;
   448	
   449				/*
   450				 * All the LMB in the set share the same NUMA
   451				 * associativity property. So read that node only once.
   452				 */
   453				if (!first) {
   454					lmb_set_nid(lmb);
   455					first = lmb;
   456				} else {
 > 457					lmb->nid = first->nid;
   458				}
   459			}
   460		}
   461	}
   462	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
field is not present in the drmem_lmb structure.

Signed-off-by: Laurent Dufour <redacted>
---
 arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..c11b6ec99ea3 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c

@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	const __be32 *p;
 	u32 i, j, lmb_sets;
 	int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+	struct drmem_lmb *first = NULL;
+#endif
 
 	lmb_sets = of_read_number(prop++, 1);
 	if (lmb_sets == 0)

@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 	for (i = 0; i < lmb_sets; i++) {
 		read_drconf_v2_cell(&dr_cell, &p);
 
+#ifdef CONFIG_MEMORY_HOTPLUG
+		/*
+		 * Fetch the NUMA node id for the fist set or if the
+		 * associativity index is different from the previous set.
+		 */
+		if (first && dr_cell.aa_index != first->aa_index)
+			first = NULL;
+#endif
+
 		for (j = 0; j < dr_cell.seq_lmbs; j++) {
 			lmb = &drmem_info->lmbs[lmb_index++];
 

@@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32 *prop)
 			lmb->aa_index = dr_cell.aa_index;
 			lmb->flags = dr_cell.flags;
 
-			lmb_set_nid(lmb);
+#ifdef CONFIG_MEMORY_HOTPLUG
+			/*
+			 * All the LMB in the set share the same NUMA
+			 * associativity property. So read that node only once.
+			 */
+			if (!first) {
+				lmb_set_nid(lmb);
+				first = lmb;
+			} else {
+				lmb->nid = first->nid;
+			}
+#endif
 		}
 	}
 }

-- 
2.28.0

Le 05/08/2020 à 12:43, kernel test robot a écrit :

Hi Laurent,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on linux/master linus/master v5.8 next-20200804]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Laurent-Dufour/powerpc-drmem-Don-t-compute-the-NUMA-node-for-each-LMB/20200805-173213
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-mpc885_ads_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
         wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
         chmod +x ~/bin/make.cross
         # save the attached .config to linux build tree
         COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=powerpc

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <redacted>

All errors (new ones prefixed by >>):

    arch/powerpc/mm/drmem.c: In function 'init_drmem_v2_lmbs':

quoted

quoted

arch/powerpc/mm/drmem.c:457:8: error: 'struct drmem_lmb' has no member named 'nid'

      457 |     lmb->nid = first->nid;
          |        ^~
    arch/powerpc/mm/drmem.c:457:21: error: 'struct drmem_lmb' has no member named 'nid'
      457 |     lmb->nid = first->nid;
          |                     ^~

My mistake, the nid member is only present when CONFIG_MEMORY_HOTPLUG is set.

I'll send a new version fixing this.

vim +457 arch/powerpc/mm/drmem.c

    397	
    398	static void __init init_drmem_v2_lmbs(const __be32 *prop)
    399	{
    400		struct drmem_lmb *lmb, *first;
    401		struct of_drconf_cell_v2 dr_cell;
    402		const __be32 *p;
    403		u32 i, j, lmb_sets;
    404		int lmb_index;
    405	
    406		lmb_sets = of_read_number(prop++, 1);
    407		if (lmb_sets == 0)
    408			return;
    409	
    410		/* first pass, calculate the number of LMBs */
    411		p = prop;
    412		for (i = 0; i < lmb_sets; i++) {
    413			read_drconf_v2_cell(&dr_cell, &p);
    414			drmem_info->n_lmbs += dr_cell.seq_lmbs;
    415		}
    416	
    417		drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
    418					   GFP_KERNEL);
    419		if (!drmem_info->lmbs)
    420			return;
    421	
    422		/* second pass, read in the LMB information */
    423		lmb_index = 0;
    424		p = prop;
    425		first = NULL;
    426	
    427		for (i = 0; i < lmb_sets; i++) {
    428			read_drconf_v2_cell(&dr_cell, &p);
    429	
    430			/*
    431			 * Fetch the NUMA node id for the fist set or if the
    432			 * associativity index is different from the previous set.
    433			 */
    434			if (first && dr_cell.aa_index != first->aa_index)
    435				first = NULL;
    436	
    437			for (j = 0; j < dr_cell.seq_lmbs; j++) {
    438				lmb = &drmem_info->lmbs[lmb_index++];
    439	
    440				lmb->base_addr = dr_cell.base_addr;
    441				dr_cell.base_addr += drmem_info->lmb_size;
    442	
    443				lmb->drc_index = dr_cell.drc_index;
    444				dr_cell.drc_index++;
    445	
    446				lmb->aa_index = dr_cell.aa_index;
    447				lmb->flags = dr_cell.flags;
    448	
    449				/*
    450				 * All the LMB in the set share the same NUMA
    451				 * associativity property. So read that node only once.
    452				 */
    453				if (!first) {
    454					lmb_set_nid(lmb);
    455					first = lmb;
    456				} else {
  > 457					lmb->nid = first->nid;
    458				}
    459			}
    460		}
    461	}
    462	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

Le 05/08/2020 à 15:35, Laurent Dufour a écrit :

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
field is not present in the drmem_lmb structure.

Signed-off-by: Laurent Dufour <redacted>

It looks like this patch was superseded by e5e179aa3a39 ("pseries/drmem: 
don't cache node id in drmem_lmb struct").

If not, anyway it conflicts with that patch so it has to be rebased.

Thanks
Christophe

On 02/04/2022, 18:35:15, Christophe Leroy wrote:

Le 05/08/2020 à 15:35, Laurent Dufour a écrit :

quoted

All the LMB from the same set of ibm,dynamic-memory-v2 property are
sharing the same NUMA node. Don't compute that node for each one.

Tested on a system with 1022 LMBs spread on 4 NUMA nodes, only 4 calls to
lmb_set_nid() have been made instead of 1022.

This should prevent some soft lockups when starting large guests

Code has meaning only if CONFIG_MEMORY_HOTPLUG is set, otherwise the nid
field is not present in the drmem_lmb structure.

Signed-off-by: Laurent Dufour <redacted>

It looks like this patch was superseded by e5e179aa3a39 ("pseries/drmem:
don't cache node id in drmem_lmb struct").

That patch has been superseded and can be dropped.

Thanks,
Laurent.

If not, anyway it conflicts with that patch so it has to be rebased.

Thanks
Christophe

quoted

---
  arch/powerpc/mm/drmem.c | 25 ++++++++++++++++++++++++-
  1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index b2eeea39684c..c11b6ec99ea3 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c

@@ -402,6 +402,9 @@ static void __init init_drmem_v2_lmbs(const __be32

*prop)
      const __be32 *p;
      u32 i, j, lmb_sets;
      int lmb_index;
+#ifdef CONFIG_MEMORY_HOTPLUG
+    struct drmem_lmb *first = NULL;
+#endif
        lmb_sets = of_read_number(prop++, 1);
      if (lmb_sets == 0)

@@ -426,6 +429,15 @@ static void __init init_drmem_v2_lmbs(const __be32

*prop)
      for (i = 0; i < lmb_sets; i++) {
          read_drconf_v2_cell(&dr_cell, &p);
  +#ifdef CONFIG_MEMORY_HOTPLUG
+        /*
+         * Fetch the NUMA node id for the fist set or if the
+         * associativity index is different from the previous set.
+         */
+        if (first && dr_cell.aa_index != first->aa_index)
+            first = NULL;
+#endif
+
          for (j = 0; j < dr_cell.seq_lmbs; j++) {
              lmb = &drmem_info->lmbs[lmb_index++];
  @@ -438,7 +450,18 @@ static void __init init_drmem_v2_lmbs(const __be32
*prop)
              lmb->aa_index = dr_cell.aa_index;
              lmb->flags = dr_cell.flags;
  -            lmb_set_nid(lmb);
+#ifdef CONFIG_MEMORY_HOTPLUG
+            /*
+             * All the LMB in the set share the same NUMA
+             * associativity property. So read that node only once.
+             */
+            if (!first) {
+                lmb_set_nid(lmb);
+                first = lmb;
+            } else {
+                lmb->nid = first->nid;
+            }
+#endif
          }
      }
  }