[RFC v2 0/3] powerpc/hotplug: Fix affinity assoc for LPAR migration

hotplug/mobility: Recognize more changes to the associativity of
memory blocks described by the 'ibm,dynamic-memory' and 'cpu'
properties when processing the topology of LPARS in Post Migration
events.  Previous efforts only recognized whether a memory block's
assignment had changed in the property.  Changes here include:

* Checking the aa_index values of the old/new properties and 'readd'
  any block for which the setting has changed.
* Checking for changes in cpu associativity and making 'readd' calls
  when differences are observed.

Signed-off-by: Michael Bringmann <redacted>
---
Changes in RFC:
  -- Simplify code to update CPU nodes during mobility checks.
     Remove functions to generate extra HP_ELOG messages in favor
     of direct function calls to dlpar_cpu_readd_by_index.
  -- Move check for "cpu" node type from pseries_update_cpu to
     pseries_smp_notifier in 'hotplug-cpu.c'
  -- Remove functions 'pseries_memory_readd_by_index' and
     'pseries_cpu_readd_by_index' as no longer needed outside of
     'mobility.c'.
---
 arch/powerpc/platforms/pseries/hotplug-cpu.c    |   69 +++++++++++++++++++++++
 arch/powerpc/platforms/pseries/hotplug-memory.c |    6 ++
 2 files changed, 75 insertions(+)

diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
index a7d14aa7..91ef22a 100644
--- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
+++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c

@@ -636,6 +636,27 @@ static int dlpar_cpu_remove_by_index(u32 drc_index)
 	return rc;
 }
 
+static int dlpar_cpu_readd_by_index(u32 drc_index)
+{
+	int rc = 0;
+
+	pr_info("Attempting to update CPU, drc index %x\n", drc_index);
+
+	if (dlpar_cpu_remove_by_index(drc_index))
+		rc = -EINVAL;
+	else if (dlpar_cpu_add(drc_index))
+		rc = -EINVAL;

+
+	if (rc)
+		pr_info("Failed to update cpu at drc_index %lx\n",
+				(unsigned long int)drc_index);
+	else
+		pr_info("CPU at drc_index %lx was updated\n",
+				(unsigned long int)drc_index);
+
+	return rc;
+}
+
 static int find_dlpar_cpus_to_remove(u32 *cpu_drcs, int cpus_to_remove)
 {
 	struct device_node *dn;

@@ -826,6 +847,9 @@ int dlpar_cpu(struct pseries_hp_errorlog *hp_elog)
 		else
 			rc = -EINVAL;
 		break;
+	case PSERIES_HP_ELOG_ACTION_READD:
+		rc = dlpar_cpu_readd_by_index(drc_index);
+		break;
 	default:
 		pr_err("Invalid action (%d) specified\n", hp_elog->action);
 		rc = -EINVAL;

@@ -876,12 +900,53 @@ static ssize_t dlpar_cpu_release(const char *buf, size_t count)
 
 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
 
+static int pseries_update_cpu(struct of_reconfig_data *pr)
+{
+	u32 old_entries, new_entries;
+	__be32 *p, *old_assoc, *new_assoc;
+	int rc = 0;
+
+	/* So far, we only handle the 'ibm,associativity' property,
+	 * here.
+	 * The first int of the property is the number of domains
+	 * described.  This is followed by an array of level values.
+	 */
+	p = (__be32 *) pr->old_prop->value;
+	if (!p)
+		return -EINVAL;
+	old_entries = be32_to_cpu(*p++);
+	old_assoc = p;
+
+	p = (__be32 *)pr->prop->value;
+	if (!p)
+		return -EINVAL;
+	new_entries = be32_to_cpu(*p++);
+	new_assoc = p;
+
+	if (old_entries == new_entries) {
+		int sz = old_entries * sizeof(int);
+
+		if (!memcmp(old_assoc, new_assoc, sz))
+			rc = dlpar_cpu_readd_by_index(
+					be32_to_cpu(pr->dn->phandle));
+
+	} else {
+		rc = dlpar_cpu_readd_by_index(
+					be32_to_cpu(pr->dn->phandle));
+	}
+
+	return rc;
+}
+
 static int pseries_smp_notifier(struct notifier_block *nb,
 				unsigned long action, void *data)
 {
 	struct of_reconfig_data *rd = data;
 	int err = 0;
 
+	if (strcmp(rd->dn->type, "cpu"))
+		return notifier_from_errno(err);
+
 	switch (action) {
 	case OF_RECONFIG_ATTACH_NODE:
 		err = pseries_add_processor(rd->dn);

@@ -889,6 +954,10 @@ static int pseries_smp_notifier(struct notifier_block *nb,
 	case OF_RECONFIG_DETACH_NODE:
 		pseries_remove_processor(rd->dn);
 		break;
+	case OF_RECONFIG_UPDATE_PROPERTY:
+		if (!strcmp(rd->prop->name, "ibm,associativity"))
+			err = pseries_update_cpu(rd);
+		break;
 	}
 	return notifier_from_errno(err);
 }

diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index c1578f5..2341eae 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c

@@ -1040,6 +1040,12 @@ static int pseries_update_drconf_memory(struct of_reconfig_data *pr)
 					  memblock_size);
 			rc = (rc < 0) ? -EINVAL : 0;
 			break;
+		} else if ((be32_to_cpu(old_drmem[i].aa_index) !=
+					be32_to_cpu(new_drmem[i].aa_index)) &&
+				(be32_to_cpu(new_drmem[i].flags) &
+					DRCONF_MEM_ASSIGNED)) {
+			rc = dlpar_memory_readd_by_index(
+				be32_to_cpu(new_drmem[i].drc_index));
 		}
 	}
 	return rc;

Signed-off-by: Michael Bringmann <redacted>
---
Changes in RFC:
  -- Simplify code to update CPU nodes during mobility checks.
     Remove functions to generate extra HP_ELOG messages in favor
     of direct function calls to dlpar_cpu_readd_by_index.
  -- Move check for "cpu" node type from pseries_update_cpu to
     pseries_smp_notifier in 'hotplug-cpu.c'
  -- Remove functions 'pseries_memory_readd_by_index' and
     'pseries_cpu_readd_by_index' as no longer needed outside of
     'mobility.c'.
---
 arch/powerpc/platforms/pseries/hotplug-cpu.c    |   69 +++++++++++++++++++++++
 arch/powerpc/platforms/pseries/hotplug-memory.c |    6 ++
 2 files changed, 75 insertions(+)

diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
index a7d14aa7..91ef22a 100644
--- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
+++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c

@@ -636,6 +636,27 @@ static int dlpar_cpu_remove_by_index(u32 drc_index)
 	return rc;
 }

+static int dlpar_cpu_readd_by_index(u32 drc_index)
+{
+	int rc = 0;
+
+	pr_info("Attempting to update CPU, drc index %x\n", drc_index);

+
+	if (dlpar_cpu_remove_by_index(drc_index))
+		rc = -EINVAL;
+	else if (dlpar_cpu_add(drc_index))
+		rc = -EINVAL;
+
+	if (rc)
+		pr_info("Failed to update cpu at drc_index %lx\n",
+				(unsigned long int)drc_index);
+	else
+		pr_info("CPU at drc_index %lx was updated\n",
+				(unsigned long int)drc_index);
+
+	return rc;
+}
+
 static int find_dlpar_cpus_to_remove(u32 *cpu_drcs, int cpus_to_remove)
 {
 	struct device_node *dn;

@@ -826,6 +847,9 @@ int dlpar_cpu(struct pseries_hp_errorlog *hp_elog)
 		else
 			rc = -EINVAL;
 		break;
+	case PSERIES_HP_ELOG_ACTION_READD:
+		rc = dlpar_cpu_readd_by_index(drc_index);
+		break;
 	default:
 		pr_err("Invalid action (%d) specified\n", hp_elog->action);
 		rc = -EINVAL;

@@ -876,12 +900,53 @@ static ssize_t dlpar_cpu_release(const char *buf, size_t count)

 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */

+static int pseries_update_cpu(struct of_reconfig_data *pr)
+{
+	u32 old_entries, new_entries;
+	__be32 *p, *old_assoc, *new_assoc;
+	int rc = 0;
+
+	/* So far, we only handle the 'ibm,associativity' property,
+	 * here.
+	 * The first int of the property is the number of domains
+	 * described.  This is followed by an array of level values.
+	 */
+	p = (__be32 *) pr->old_prop->value;
+	if (!p)
+		return -EINVAL;
+	old_entries = be32_to_cpu(*p++);
+	old_assoc = p;
+
+	p = (__be32 *)pr->prop->value;
+	if (!p)
+		return -EINVAL;
+	new_entries = be32_to_cpu(*p++);
+	new_assoc = p;
+
+	if (old_entries == new_entries) {
+		int sz = old_entries * sizeof(int);
+
+		if (!memcmp(old_assoc, new_assoc, sz))
+			rc = dlpar_cpu_readd_by_index(
+					be32_to_cpu(pr->dn->phandle));
+
+	} else {
+		rc = dlpar_cpu_readd_by_index(
+					be32_to_cpu(pr->dn->phandle));
+	}
+
+	return rc;
+}

+
 static int pseries_smp_notifier(struct notifier_block *nb,
 				unsigned long action, void *data)
 {
 	struct of_reconfig_data *rd = data;
 	int err = 0;

+	if (strcmp(rd->dn->type, "cpu"))
+		return notifier_from_errno(err);
+
 	switch (action) {
 	case OF_RECONFIG_ATTACH_NODE:
 		err = pseries_add_processor(rd->dn);

@@ -889,6 +954,10 @@ static int pseries_smp_notifier(struct notifier_block *nb,
 	case OF_RECONFIG_DETACH_NODE:
 		pseries_remove_processor(rd->dn);
 		break;
+	case OF_RECONFIG_UPDATE_PROPERTY:
+		if (!strcmp(rd->prop->name, "ibm,associativity"))
+			err = pseries_update_cpu(rd);
+		break;
 	}
 	return notifier_from_errno(err);
 }

diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index c1578f5..2341eae 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c

@@ -1040,6 +1040,12 @@ static int pseries_update_drconf_memory(struct of_reconfig_data *pr)
 					  memblock_size);
 			rc = (rc < 0) ? -EINVAL : 0;
 			break;
+		} else if ((be32_to_cpu(old_drmem[i].aa_index) !=
+					be32_to_cpu(new_drmem[i].aa_index)) &&
+				(be32_to_cpu(new_drmem[i].flags) &
+					DRCONF_MEM_ASSIGNED)) {
+			rc = dlpar_memory_readd_by_index(
+				be32_to_cpu(new_drmem[i].drc_index))>  		}
 	}
 	return rc;

postmigration/memory: In an LPAR migration scenario, the property
"ibm,associativity-lookup-arrays" may change.  In the event that a
row of the array differs, locate all assigned memory blocks with that
'aa_index' and 're-add' them to the system memory block data structures.
In the process of the 're-add', the appropriate entry of the property
'ibm,dynamic-memory' would be updated as well as any other applicable
system data structures.

Signed-off-by: Michael Bringmann <redacted>
---
Changes in RFC v2:
  -- Simplify code to update memory nodes during mobility checks.
     Remove functions to generate extra HP_ELOG messages in favor
     of direct function calls to dlpar_memory_readd_by_index.
---
 arch/powerpc/platforms/pseries/hotplug-memory.c |  120 +++++++++++++++++++++++
 1 file changed, 120 insertions(+)

diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index 2341eae..b63181d 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c

@@ -1051,6 +1051,123 @@ static int pseries_update_drconf_memory(struct of_reconfig_data *pr)
 	return rc;
 }

+struct assoc_arrays {
+	u32 n_arrays;
+	u32 array_sz;
+	const __be32 *arrays;
+};
+
+static int pseries_update_ala_memory_aai(int aa_index,
+					struct property *dmprop)
+{
+	struct of_drconf_cell *drmem;
+	u32 entries;
+	__be32 *p;
+	int i;
+	int rc = 0;
+
+	p = (__be32 *) dmprop->value;
+	if (!p)
+		return -EINVAL;
+
+	/* The first int of the property is the number of lmb's
+	 * described by the property. This is followed by an array
+	 * of of_drconf_cell entries. Get the number of entries
+	 * and skip to the array of of_drconf_cell's.
+	 */
+	entries = be32_to_cpu(*p++);
+	drmem = (struct of_drconf_cell *)p;
+
+	for (i = 0; i < entries; i++) {
+		if ((be32_to_cpu(drmem[i].aa_index) != aa_index) &&
+			(be32_to_cpu(drmem[i].flags) & DRCONF_MEM_ASSIGNED)) {
+			rc = dlpar_memory_readd_by_index(
+				be32_to_cpu(drmem[i].drc_index));
+		}
+	}
+
+	return rc;
+}
+
+static int pseries_update_ala_memory(struct of_reconfig_data *pr)
+{
+	struct assoc_arrays new_ala, old_ala;
+	struct device_node *dn;
+	struct property *dmprop;
+	__be32 *p;
+	int i, lim;
+
+	if (rtas_hp_event)
+		return 0;
+
+	dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
+	if (!dn)
+		return -ENODEV;
+
+	dmprop = of_find_property(dn, "ibm,dynamic-memory", NULL);
+	if (!dmprop) {
+		of_node_put(dn);
+		return -ENODEV;
+	}
+
+	/*
+	 * The layout of the ibm,associativity-lookup-arrays
+	 * property is a number N indicating the number of
+	 * associativity arrays, followed by a number M
+	 * indicating the size of each associativity array,
+	 * followed by a list of N associativity arrays.
+	 */
+
+	p = (__be32 *) pr->old_prop->value;
+	if (!p) {
+		of_node_put(dn);
+		return -EINVAL;
+	}
+	old_ala.n_arrays = of_read_number(p++, 1);
+	old_ala.array_sz = of_read_number(p++, 1);
+	old_ala.arrays = p;
+
+	p = (__be32 *) pr->prop->value;
+	if (!p) {
+		of_node_put(dn);
+		return -EINVAL;
+	}
+	new_ala.n_arrays = of_read_number(p++, 1);
+	new_ala.array_sz = of_read_number(p++, 1);
+	new_ala.arrays = p;
+
+	lim = (new_ala.n_arrays > old_ala.n_arrays) ? old_ala.n_arrays :
+			new_ala.n_arrays;
+
+	if (old_ala.array_sz == new_ala.array_sz) {
+
+		for (i = 0; i < lim; i++) {
+			int index = (i * new_ala.array_sz);
+
+			if (!memcmp(&old_ala.arrays[index],
+						&new_ala.arrays[index],
+						new_ala.array_sz))
+				continue;
+
+			pseries_update_ala_memory_aai(i, dmprop);
+		}
+
+		for (i = lim; i < new_ala.n_arrays; i++)
+			pseries_update_ala_memory_aai(i, dmprop);
+
+	} else {
+		/* Update all entries representing these rows;
+		 * as all rows have different sizes, none can
+		 * have equivalent values.
+		 */
+		for (i = 0; i < lim; i++)
+			pseries_update_ala_memory_aai(i, dmprop);
+	}
+
+	of_node_put(dn);
+	return 0;
+}

+
 static int pseries_memory_notifier(struct notifier_block *nb,
 				   unsigned long action, void *data)
 {

@@ -1067,6 +1184,9 @@ static int pseries_memory_notifier(struct notifier_block *nb,
 	case OF_RECONFIG_UPDATE_PROPERTY:
 		if (!strcmp(rd->prop->name, "ibm,dynamic-memory"))
 			err = pseries_update_drconf_memory(rd);
+		if (!strcmp(rd->prop->name,
+				"ibm,associativity-lookup-arrays"))
+			err = pseries_update_ala_memory(rd);
 		break;
 	}
 	return notifier_from_errno(err);

Signed-off-by: Michael Bringmann <redacted>
---
Changes in RFC v2:
  -- Reuse existing parser code from 'drmem.c' in parsing property
     'imb,dynamic-memory-v2' for migration.
  -- Fix crash during migration that occurs on non-VPHN systems
     when attempting to reset topology timer.
  -- Change section of a support function + variable from __init 
     to normal runtime to make them visible to migration code.
---
 arch/powerpc/include/asm/drmem.h                |    8 +
 arch/powerpc/mm/drmem.c                         |   23 ++-
 arch/powerpc/mm/numa.c                          |    3 
 arch/powerpc/platforms/pseries/hotplug-memory.c |  175 +++++++++++++++++++----
 drivers/of/fdt.c                                |    4 -
 include/linux/of_fdt.h                          |    2 
 6 files changed, 170 insertions(+), 45 deletions(-)

diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h
index 47a7012..e4773c9 100644
--- a/arch/powerpc/include/asm/drmem.h
+++ b/arch/powerpc/include/asm/drmem.h

@@ -92,6 +92,14 @@ void __init walk_drmem_lmbs(struct device_node *dn,
 			void (*func)(struct drmem_lmb *, const __be32 **));
 int drmem_update_dt(void);

+void walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *data,
+			void (*func)(struct drmem_lmb *, const __be32 **));
+
+void read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
+			const __be32 **prop);
+void walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *data,
+			void (*func)(struct drmem_lmb *, const __be32 **));
+
 #ifdef CONFIG_PPC_PSERIES
 void __init walk_drmem_lmbs_early(unsigned long node,
 			void (*func)(struct drmem_lmb *, const __be32 **));

diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
index 31dbe14..e47a6e0 100644
--- a/arch/powerpc/mm/drmem.c
+++ b/arch/powerpc/mm/drmem.c

@@ -192,7 +192,7 @@ int drmem_update_dt(void)
 	return rc;
 }

-static void __init read_drconf_v1_cell(struct drmem_lmb *lmb,
+static void read_drconf_v1_cell(struct drmem_lmb *lmb,
 				       const __be32 **prop)
 {
 	const __be32 *p = *prop;

@@ -208,7 +208,7 @@ static void __init read_drconf_v1_cell(struct drmem_lmb *lmb,
 	*prop = p;
 }

-static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm,
+void walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *data,
 			void (*func)(struct drmem_lmb *, const __be32 **))
 {
 	struct drmem_lmb lmb;

@@ -218,11 +218,12 @@ static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm,

 	for (i = 0; i < n_lmbs; i++) {
 		read_drconf_v1_cell(&lmb, &prop);
-		func(&lmb, &usm);
+		func(&lmb, &data);
 	}
 }
+EXPORT_SYMBOL(walk_drmem_v1_lmbs);

-static void __init read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
+void read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
 				       const __be32 **prop)
 {
 	const __be32 *p = *prop;

@@ -235,8 +236,9 @@ static void __init read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,

 	*prop = p;
 }
+EXPORT_SYMBOL(read_drconf_v2_cell);

-static void __init __walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *usm,
+void walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *data,
 			void (*func)(struct drmem_lmb *, const __be32 **))
 {
 	struct of_drconf_cell_v2 dr_cell;

@@ -258,10 +260,11 @@ static void __init __walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *usm,
 			lmb.aa_index = dr_cell.aa_index;
 			lmb.flags = dr_cell.flags;

-			func(&lmb, &usm);
+			func(&lmb, &data);
 		}
 	}
 }
+EXPORT_SYMBOL(walk_drmem_v2_lmbs);

 #ifdef CONFIG_PPC_PSERIES
 void __init walk_drmem_lmbs_early(unsigned long node,

@@ -280,12 +283,12 @@ void __init walk_drmem_lmbs_early(unsigned long node,

 	prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &len);
 	if (prop) {
-		__walk_drmem_v1_lmbs(prop, usm, func);
+		walk_drmem_v1_lmbs(prop, usm, func);
 	} else {
 		prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory-v2",
 					   &len);
 		if (prop)
-			__walk_drmem_v2_lmbs(prop, usm, func);
+			walk_drmem_v2_lmbs(prop, usm, func);
 	}

 	memblock_dump_all();

@@ -340,11 +343,11 @@ void __init walk_drmem_lmbs(struct device_node *dn,

 	prop = of_get_property(dn, "ibm,dynamic-memory", NULL);
 	if (prop) {
-		__walk_drmem_v1_lmbs(prop, usm, func);
+		walk_drmem_v1_lmbs(prop, usm, func);
 	} else {
 		prop = of_get_property(dn, "ibm,dynamic-memory-v2", NULL);
 		if (prop)
-			__walk_drmem_v2_lmbs(prop, usm, func);
+			walk_drmem_v2_lmbs(prop, usm, func);
 	}
 }

diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 0e573f9..2545fea 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c

@@ -1395,7 +1395,8 @@ static void topology_timer_fn(struct timer_list *unused)

 static void reset_topology_timer(void)
 {
-	mod_timer(&topology_timer, jiffies + topology_timer_secs * HZ);
+	if (vphn_enabled)
+		mod_timer(&topology_timer, jiffies + topology_timer_secs * HZ);