[PATCH 0/3] hotplug memory remove updates

Hi Paul,

Here are the hotplug memory remove updates for 2.6.25-rc2-mm1.

[PATCH 1/3] ppc64-specific remove htab bolted mapping support
[PATCH 2/3] generic __remove_pages() support
[PATCH 3/3] ppc64-specific memory notifier support

As you can see PATCH 1,3 are ppc64-specific. PATCH 1 was already
reviewed by you. Could you please review PATCH 3 and let me know,
if I missing something ?

I would like to submit these for -mm inclusion after your review.

Thanks,
Badari

For memory remove, we need to clean up htab mappings for the
section of the memory we are removing.

This patch implements support for removing htab bolted mappings
for ppc64 lpar. Other sub-archs, may need to implement similar
functionality for the hotplug memory remove to work. 

Signed-off-by: Badari Pulavarty <redacted>
Acked-by: Paul Mackerras <redacted>
---
 arch/powerpc/mm/hash_utils_64.c       |   26 ++++++++++++++++++++++++++
 arch/powerpc/platforms/pseries/lpar.c |   15 +++++++++++++++
 include/asm-powerpc/machdep.h         |    2 ++
 include/asm-powerpc/sparsemem.h       |    1 +
 5 files changed, 44 insertions(+), 4 deletions(-)

Index: linux-2.6.25-rc2/arch/powerpc/mm/hash_utils_64.c
===================================================================

--- linux-2.6.25-rc2.orig/arch/powerpc/mm/hash_utils_64.c	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/arch/powerpc/mm/hash_utils_64.c	2008-02-27 12:59:53.000000000 -0800

@@ -191,6 +191,26 @@ int htab_bolt_mapping(unsigned long vsta
 	return ret < 0 ? ret : 0;
 }
 
+static int htab_remove_mapping(unsigned long vstart, unsigned long vend,
+		      int psize, int ssize)
+{
+	unsigned long vaddr;
+	unsigned int step, shift;
+
+	shift = mmu_psize_defs[psize].shift;
+	step = 1 << shift;
+
+	if (!ppc_md.hpte_removebolted) {
+		printk("Sub-arch doesn't implement hpte_removebolted\n");
+		return -EINVAL;
+	}
+
+	for (vaddr = vstart; vaddr < vend; vaddr += step)
+		ppc_md.hpte_removebolted(vaddr, psize, ssize);
+
+	return 0;
+}
+
 static int __init htab_dt_scan_seg_sizes(unsigned long node,
 					 const char *uname, int depth,
 					 void *data)

@@ -429,6 +449,12 @@ void create_section_mapping(unsigned lon
 			_PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX,
 			mmu_linear_psize, mmu_kernel_ssize));
 }
+
+int remove_section_mapping(unsigned long start, unsigned long end)
+{
+	return htab_remove_mapping(start, end, mmu_linear_psize,
+				mmu_kernel_ssize);
+}
 #endif /* CONFIG_MEMORY_HOTPLUG */
 
 static inline void make_bl(unsigned int *insn_addr, void *func)

Index: linux-2.6.25-rc2/include/asm-powerpc/sparsemem.h
===================================================================

--- linux-2.6.25-rc2.orig/include/asm-powerpc/sparsemem.h	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/include/asm-powerpc/sparsemem.h	2008-02-27 12:59:53.000000000 -0800

@@ -15,6 +15,7 @@
 
 #ifdef CONFIG_MEMORY_HOTPLUG
 extern void create_section_mapping(unsigned long start, unsigned long end);
+extern int remove_section_mapping(unsigned long start, unsigned long end);
 #ifdef CONFIG_NUMA
 extern int hot_add_scn_to_nid(unsigned long scn_addr);
 #else

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/lpar.c
===================================================================

--- linux-2.6.25-rc2.orig/arch/powerpc/platforms/pseries/lpar.c	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/lpar.c	2008-02-27 12:59:53.000000000 -0800

@@ -520,6 +520,20 @@ static void pSeries_lpar_hpte_invalidate
 	BUG_ON(lpar_rc != H_SUCCESS);
 }
 
+static void pSeries_lpar_hpte_removebolted(unsigned long ea,
+					   int psize, int ssize)
+{
+	unsigned long slot, vsid, va;
+
+	vsid = get_kernel_vsid(ea, ssize);
+	va = hpt_va(ea, vsid, ssize);
+
+	slot = pSeries_lpar_hpte_find(va, psize, ssize);
+	BUG_ON(slot == -1);
+
+	pSeries_lpar_hpte_invalidate(slot, va, psize, ssize, 0);
+}
+
 /* Flag bits for H_BULK_REMOVE */
 #define HBR_REQUEST	0x4000000000000000UL
 #define HBR_RESPONSE	0x8000000000000000UL

@@ -597,6 +611,7 @@ void __init hpte_init_lpar(void)
 	ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
 	ppc_md.hpte_insert	= pSeries_lpar_hpte_insert;
 	ppc_md.hpte_remove	= pSeries_lpar_hpte_remove;
+	ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted;
 	ppc_md.flush_hash_range	= pSeries_lpar_flush_hash_range;
 	ppc_md.hpte_clear_all   = pSeries_lpar_hptab_clear;
 }

Index: linux-2.6.25-rc2/include/asm-powerpc/machdep.h
===================================================================

--- linux-2.6.25-rc2.orig/include/asm-powerpc/machdep.h	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/include/asm-powerpc/machdep.h	2008-02-27 12:59:53.000000000 -0800

@@ -68,6 +68,8 @@ struct machdep_calls {
 				       unsigned long vflags,
 				       int psize, int ssize);
 	long		(*hpte_remove)(unsigned long hpte_group);
+	void            (*hpte_removebolted)(unsigned long ea,
+					       int psize, int ssize);
 	void		(*flush_hash_range)(unsigned long number, int local);
 
 	/* special for kexec, to be called in real mode, linar mapping is

Generic helper function to remove section mappings and sysfs entries
for the section of the memory we are removing.  offline_pages() correctly 
adjusted zone and marked the pages reserved.

Issue: If mem_map, usemap allocation could come from different places -
kmalloc, vmalloc, alloc_pages or bootmem. There is no easy way
to find and free up properly. Especially for bootmem, we need to
know which node the allocation came from.

Signed-off-by: Badari Pulavarty <redacted>

---
 include/linux/memory_hotplug.h |    4 +++
 mm/memory_hotplug.c            |   44 +++++++++++++++++++++++++++++++++++++++++
 mm/sparse.c                    |   43 +++++++++++++++++++++++++++++++++++++---
 3 files changed, 88 insertions(+), 3 deletions(-)

Index: linux-2.6.25-rc2/mm/memory_hotplug.c
===================================================================

--- linux-2.6.25-rc2.orig/mm/memory_hotplug.c	2008-02-27 12:58:17.000000000 -0800
+++ linux-2.6.25-rc2/mm/memory_hotplug.c	2008-02-27 16:06:50.000000000 -0800

@@ -102,6 +102,21 @@ static int __add_section(struct zone *zo
 	return register_new_memory(__pfn_to_section(phys_start_pfn));
 }
 
+static int __remove_section(struct zone *zone, struct mem_section *ms)
+{
+	int ret = -EINVAL;
+
+	if (!valid_section(ms))
+		return ret;
+
+	ret = unregister_memory_section(ms);
+	if (ret)
+		return ret;
+
+	sparse_remove_one_section(zone, ms);
+	return 0;
+}
+
 /*
  * Reasonably generic function for adding memory.  It is
  * expected that archs that support memory hotplug will

@@ -135,6 +150,35 @@ int __add_pages(struct zone *zone, unsig
 }
 EXPORT_SYMBOL_GPL(__add_pages);
 
+int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
+		 unsigned long nr_pages)
+{
+	unsigned long i, ret = 0;
+	int sections_to_remove;
+	unsigned long flags;
+	struct pglist_data *pgdat = zone->zone_pgdat;
+
+	/*
+	 * We can only remove entire sections
+	 */
+	BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
+	BUG_ON(nr_pages % PAGES_PER_SECTION);
+
+	release_mem_region(phys_start_pfn << PAGE_SHIFT, nr_pages * PAGE_SIZE);
+
+	sections_to_remove = nr_pages / PAGES_PER_SECTION;
+	for (i = 0; i < sections_to_remove; i++) {
+		unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
+		pgdat_resize_lock(pgdat, &flags);
+		ret = __remove_section(zone, __pfn_to_section(pfn));
+		pgdat_resize_unlock(pgdat, &flags);
+		if (ret)
+			break;
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(__remove_pages);
+
 static void grow_zone_span(struct zone *zone,
 		unsigned long start_pfn, unsigned long end_pfn)
 {

Index: linux-2.6.25-rc2/mm/sparse.c
===================================================================

--- linux-2.6.25-rc2.orig/mm/sparse.c	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/mm/sparse.c	2008-02-27 13:02:51.000000000 -0800

@@ -198,12 +198,13 @@ static unsigned long sparse_encode_mem_m
 }
 
 /*
- * We need this if we ever free the mem_maps.  While not implemented yet,
- * this function is included for parity with its sibling.
+ * Decode mem_map from the coded memmap
  */
-static __attribute((unused))
+static
 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
 {
+	/* mask off the extra low bits of information */
+	coded_mem_map &= SECTION_MAP_MASK;
 	return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
 }
 

@@ -363,6 +364,26 @@ static void __kfree_section_memmap(struc
 }
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
 
+static void free_section_usemap(struct page *memmap, unsigned long *usemap)
+{
+	if (!usemap)
+		return;
+
+	/*
+	 * Check to see if allocation came from hot-plug-add
+	 */
+	if (PageSlab(virt_to_page(usemap))) {
+		kfree(usemap);
+		if (memmap)
+			__kfree_section_memmap(memmap, PAGES_PER_SECTION);
+		return;
+	}
+
+	/*
+	 * Allocations came from bootmem - how do I free up ?
+	 */
+}
+
 /*
  * returns the number of sections whose mem_maps were properly
  * set.  If this is <=0, then that means that the passed-in

@@ -415,4 +436,20 @@ out:
 	}
 	return ret;
 }
+
+void sparse_remove_one_section(struct zone *zone, struct mem_section *ms)
+{
+	struct page *memmap = NULL;
+	unsigned long *usemap = NULL;
+
+	if (ms->section_mem_map) {
+		usemap = ms->pageblock_flags;
+		memmap = sparse_decode_mem_map(ms->section_mem_map,
+						__section_nr(ms));
+		ms->section_mem_map = 0;
+		ms->pageblock_flags = NULL;
+	}
+
+	free_section_usemap(memmap, usemap);
+}
 #endif

Index: linux-2.6.25-rc2/include/linux/memory_hotplug.h
===================================================================

--- linux-2.6.25-rc2.orig/include/linux/memory_hotplug.h	2008-02-27 12:58:17.000000000 -0800
+++ linux-2.6.25-rc2/include/linux/memory_hotplug.h	2008-02-27 13:00:04.000000000 -0800

@@ -8,6 +8,7 @@
 struct page;
 struct zone;
 struct pglist_data;
+struct mem_section;
 
 #ifdef CONFIG_MEMORY_HOTPLUG
 /*

@@ -64,6 +65,8 @@ extern int offline_pages(unsigned long, 
 /* reasonably generic interface to expand the physical pages in a zone  */
 extern int __add_pages(struct zone *zone, unsigned long start_pfn,
 	unsigned long nr_pages);
+extern int __remove_pages(struct zone *zone, unsigned long start_pfn,
+	unsigned long nr_pages);
 
 /*
  * Walk thorugh all memory which is registered as resource.

@@ -188,5 +191,6 @@ extern int arch_add_memory(int nid, u64 
 extern int remove_memory(u64 start, u64 size);
 extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
 								int nr_pages);
+extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms);
 
 #endif /* __LINUX_MEMORY_HOTPLUG_H */

Hotplug memory notifier for ppc64. This gets invoked by writing
the device-node that needs to be removed to /proc/ppc64/ofdt.
We need to adjust the sections and remove sysfs entries by
calling __remove_pages(). Then call arch specific code to
get rid of htab mappings for the section of memory.

Signed-off-by: Badari Pulavarty <redacted>
---
 arch/powerpc/platforms/pseries/Makefile         |    1 
 arch/powerpc/platforms/pseries/hotplug-memory.c |   98 ++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c
===================================================================

--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-28 08:20:14.000000000 -0800

@@ -0,0 +1,98 @@
+/*
+ * pseries Memory Hotplug infrastructure.
+ *
+ * Copyright (C) 2008 Badari Pulavarty, IBM Corporation
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ */
+
+#include <asm/prom.h>
+#include <asm/firmware.h>
+#include <asm/machdep.h>
+#include <asm/pSeries_reconfig.h>
+
+static int pseries_remove_memory(struct device_node *np)
+{
+	const char *type;
+	const unsigned int *my_index;
+	const unsigned int *regs;
+	u64 start_pfn, start;
+	struct zone *zone;
+	int ret = -EINVAL;
+
+	/*
+	 * Check to see if we are actually removing memory
+	 */
+	type = of_get_property(np, "device_type", NULL);
+	if (type == NULL || strcmp(type, "memory") != 0)
+		return 0;
+
+	/*
+	 * Find the memory index and size of the removing section
+	 */
+	my_index = of_get_property(np, "ibm,my-drc-index", NULL);
+	if (!my_index)
+		return ret;
+
+	regs = of_get_property(np, "reg", NULL);
+	if (!regs)
+		return ret;
+
+	start_pfn = section_nr_to_pfn(*my_index & 0xffff);
+	zone = page_zone(pfn_to_page(start_pfn));
+
+	/*
+	 * Remove section mappings and sysfs entries for the
+	 * section of the memory we are removing.
+	 *
+	 * NOTE: Ideally, this should be done in generic code like
+	 * remove_memory(). But remove_memory() gets called by writing
+	 * to sysfs "state" file and we can't remove sysfs entries
+	 * while writing to it. So we have to defer it to here.
+	 */
+	ret = __remove_pages(zone, start_pfn, regs[3] >> PAGE_SHIFT);
+	if (ret)
+		return ret;
+
+	/*
+	 * Remove htab bolted mappings for this section of memory
+	 */
+ 	start = (unsigned long)__va(start_pfn << PAGE_SHIFT);
+ 	ret = remove_section_mapping(start, start + regs[3]);
+	return ret;
+}
+
+static int pseries_memory_notifier(struct notifier_block *nb,
+				unsigned long action, void *node)
+{
+	int err = NOTIFY_OK;
+
+	switch (action) {
+	case PSERIES_RECONFIG_ADD:
+		break;
+	case PSERIES_RECONFIG_REMOVE:
+		if (pseries_remove_memory(node))
+			err = NOTIFY_BAD;
+		break;
+	default:
+		err = NOTIFY_DONE;
+		break;
+	}
+	return err;
+}
+
+static struct notifier_block pseries_smp_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/Makefile
===================================================================

--- linux-2.6.25-rc2.orig/arch/powerpc/platforms/pseries/Makefile	2008-02-28 08:15:53.000000000 -0800
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/Makefile	2008-02-28 08:17:57.000000000 -0800

@@ -14,6 +14,7 @@ obj-$(CONFIG_PCI)	+= pci.o pci_dlpar.o
 obj-$(CONFIG_PCI_MSI)	+= msi.o
 
 obj-$(CONFIG_HOTPLUG_CPU)	+= hotplug-cpu.o
+obj-$(CONFIG_MEMORY_HOTPLUG)	+= hotplug-memory.o
 
 obj-$(CONFIG_HVC_CONSOLE)	+= hvconsole.o
 obj-$(CONFIG_HVCS)		+= hvcserver.o

Badari Pulavarty wrote:

+static struct notifier_block pseries_smp_nb = {

Rename this to pseries_mem_nb?

+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

arch_initcall doesn't seem appropriate.  __initcall should be fine.

On Thu, 2008-02-28 at 11:20 -0600, Nathan Lynch wrote:

Badari Pulavarty wrote:

quoted

+static struct notifier_block pseries_smp_nb = {

Rename this to pseries_mem_nb?

Sure.

quoted

+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

arch_initcall doesn't seem appropriate.  __initcall should be fine.

Okay, if you say so :)

I based the code on arch/powerpc/platforms/pseries/hotplug-cpu.c 

Thanks,
Badari

On Thu, 2008-02-28 at 08:46 -0800, Badari Pulavarty wrote:

+
+static struct notifier_block pseries_smp_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

This is going to fire on non-pseries LPAR platforms, like iSeries and
PS3. Which is not what you want I think.

cheers

-- 
Michael Ellerman
OzLabs, IBM Australia Development Lab

wwweb: http://michael.ellerman.id.au
phone: +61 2 6212 1183 (tie line 70 21183)

We do not inherit the earth from our ancestors,
we borrow it from our children. - S.M.A.R.T Person

On Thu, 2008-02-28 at 18:39 -0600, Nathan Lynch wrote:

Michael Ellerman wrote:

quoted

On Thu, 2008-02-28 at 08:46 -0800, Badari Pulavarty wrote:

quoted

Hotplug memory notifier for ppc64. This gets invoked by writing
the device-node that needs to be removed to /proc/ppc64/ofdt.
We need to adjust the sections and remove sysfs entries by
calling __remove_pages(). Then call arch specific code to
get rid of htab mappings for the section of memory.

Signed-off-by: Badari Pulavarty <redacted>
---
 arch/powerpc/platforms/pseries/Makefile         |    1 
 arch/powerpc/platforms/pseries/hotplug-memory.c |   98 ++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c
===================================================================

--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-28 08:20:14.000000000 -0800

quoted

+
+static struct notifier_block pseries_smp_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

This is going to fire on non-pseries LPAR platforms, like iSeries and
PS3. Which is not what you want I think.

Well, the notifier will be registered, yes, but it will never be
called because that path is reachable only from a write to
/proc/ppc64/ofdt, which is not created on non-pseries.

Sure. Still seems better not to register it in the first place.

Maybe it should be

machine_device_initcall(pseries, pseries_memory_hotplug_init);

I think so.

(and pseries_cpu_hotplug_init in hotplug-cpu.c should be changed to
machine_arch_initcall)

Yeah I noticed that was not guarded as well, and I think I'm culpable
for that :)

cheers

-- 
Michael Ellerman
OzLabs, IBM Australia Development Lab

wwweb: http://michael.ellerman.id.au
phone: +61 2 6212 1183 (tie line 70 21183)

We do not inherit the earth from our ancestors,
we borrow it from our children. - S.M.A.R.T Person

Badari Pulavarty wrote:

Hi Paul,

Here are the hotplug memory remove updates for 2.6.25-rc2-mm1.

How have these been tested?  Have you initiated a memory remove
operation from the HMC?  That's the only way to catch some bugs...

Hi.

Nathan Lynch wrote:

Badari Pulavarty wrote:

quoted

Hi Paul,

Here are the hotplug memory remove updates for 2.6.25-rc2-mm1.

How have these been tested?  Have you initiated a memory remove
operation from the HMC?  That's the only way to catch some bugs...

I'm wondering how the memory hot un-plug is initiated on the pseries.
Could you tell me about this HMC?  Is it an application running in
the lpar, or is it an external entity?

Is there a 'standard' interface from userspace that can be used to
trigger the hot-unplug sequence?  I'm asking because PS3's lv1
hypervisor supports hot un-plug of memory, but it would need to be
triggered from some kind of management application running in in
userspace.

-Geoff

Nathan Lynch wrote:

Badari Pulavarty wrote:
  

quoted

Hi Paul,

Here are the hotplug memory remove updates for 2.6.25-rc2-mm1.
    

How have these been tested?  Have you initiated a memory remove
operation from the HMC?  That's the only way to catch some bugs...
  

Yes, They are testing from HMC + running "drmgr" command manually to 
hash out
all the issues.

Thanks,
Badari

On Fri, 2008-02-29 at 12:03 +1100, Michael Ellerman wrote:

On Thu, 2008-02-28 at 18:39 -0600, Nathan Lynch wrote:

quoted

Michael Ellerman wrote:

quoted

On Thu, 2008-02-28 at 08:46 -0800, Badari Pulavarty wrote:

quoted

Hotplug memory notifier for ppc64. This gets invoked by writing
the device-node that needs to be removed to /proc/ppc64/ofdt.
We need to adjust the sections and remove sysfs entries by
calling __remove_pages(). Then call arch specific code to
get rid of htab mappings for the section of memory.

Signed-off-by: Badari Pulavarty <redacted>
---
 arch/powerpc/platforms/pseries/Makefile         |    1 
 arch/powerpc/platforms/pseries/hotplug-memory.c |   98 ++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c
===================================================================

--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-28 08:20:14.000000000 -0800

quoted

+
+static struct notifier_block pseries_smp_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

This is going to fire on non-pseries LPAR platforms, like iSeries and
PS3. Which is not what you want I think.

Well, the notifier will be registered, yes, but it will never be
called because that path is reachable only from a write to
/proc/ppc64/ofdt, which is not created on non-pseries.

Sure. Still seems better not to register it in the first place.

quoted

Maybe it should be

machine_device_initcall(pseries, pseries_memory_hotplug_init);

I think so.

Here is the latest for review. (just to make sure I didn't miss
anything).

Thanks,
Badari

Hotplug memory notifier for ppc64. This gets invoked by writing
the device-node that needs to be removed to /proc/ppc64/ofdt.
We need to adjust the sections and remove sysfs entries by
calling __remove_pages(). Then call arch specific code to
get rid of htab mappings for the section of memory.

Signed-off-by: Badari Pulavarty <redacted>
---
 arch/powerpc/platforms/pseries/Makefile         |    1 
 arch/powerpc/platforms/pseries/hotplug-memory.c |   98 ++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c
===================================================================

--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-29 09:25:14.000000000 -0800

@@ -0,0 +1,98 @@
+/*
+ * pseries Memory Hotplug infrastructure.
+ *
+ * Copyright (C) 2008 Badari Pulavarty, IBM Corporation
+ *
+ *      This program is free software; you can redistribute it and/or
+ *      modify it under the terms of the GNU General Public License
+ *      as published by the Free Software Foundation; either version
+ *      2 of the License, or (at your option) any later version.
+ */
+
+#include <asm/prom.h>
+#include <asm/firmware.h>
+#include <asm/machdep.h>
+#include <asm/pSeries_reconfig.h>
+
+static int pseries_remove_memory(struct device_node *np)
+{
+	const char *type;
+	const unsigned int *my_index;
+	const unsigned int *regs;
+	u64 start_pfn, start;
+	struct zone *zone;
+	int ret = -EINVAL;
+
+	/*
+	 * Check to see if we are actually removing memory
+	 */
+	type = of_get_property(np, "device_type", NULL);
+	if (type == NULL || strcmp(type, "memory") != 0)
+		return 0;
+
+	/*
+	 * Find the memory index and size of the removing section
+	 */
+	my_index = of_get_property(np, "ibm,my-drc-index", NULL);
+	if (!my_index)
+		return ret;
+
+	regs = of_get_property(np, "reg", NULL);
+	if (!regs)
+		return ret;
+
+	start_pfn = section_nr_to_pfn(*my_index & 0xffff);
+	zone = page_zone(pfn_to_page(start_pfn));
+
+	/*
+	 * Remove section mappings and sysfs entries for the
+	 * section of the memory we are removing.
+	 *
+	 * NOTE: Ideally, this should be done in generic code like
+	 * remove_memory(). But remove_memory() gets called by writing
+	 * to sysfs "state" file and we can't remove sysfs entries
+	 * while writing to it. So we have to defer it to here.
+	 */
+	ret = __remove_pages(zone, start_pfn, regs[3] >> PAGE_SHIFT);
+	if (ret)
+		return ret;
+
+	/*
+	 * Remove htab bolted mappings for this section of memory
+	 */
+ 	start = (unsigned long)__va(start_pfn << PAGE_SHIFT);
+ 	ret = remove_section_mapping(start, start + regs[3]);
+	return ret;
+}
+
+static int pseries_memory_notifier(struct notifier_block *nb,
+				unsigned long action, void *node)
+{
+	int err = NOTIFY_OK;
+
+	switch (action) {
+	case PSERIES_RECONFIG_ADD:
+		break;
+	case PSERIES_RECONFIG_REMOVE:
+		if (pseries_remove_memory(node))
+			err = NOTIFY_BAD;
+		break;
+	default:
+		err = NOTIFY_DONE;
+		break;
+	}
+	return err;
+}
+
+static struct notifier_block pseries_mem_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_mem_nb);
+
+	return 0;
+}
+machine_device_initcall(pseries, pseries_memory_hotplug_init);

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/Makefile
===================================================================

--- linux-2.6.25-rc2.orig/arch/powerpc/platforms/pseries/Makefile	2008-02-28 08:15:53.000000000 -0800
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/Makefile	2008-02-28 08:17:57.000000000 -0800

@@ -14,6 +14,7 @@ obj-$(CONFIG_PCI)	+= pci.o pci_dlpar.o
 obj-$(CONFIG_PCI_MSI)	+= msi.o
 
 obj-$(CONFIG_HOTPLUG_CPU)	+= hotplug-cpu.o
+obj-$(CONFIG_MEMORY_HOTPLUG)	+= hotplug-memory.o
 
 obj-$(CONFIG_HVC_CONSOLE)	+= hvconsole.o
 obj-$(CONFIG_HVCS)		+= hvcserver.o

Hi,

I am wondering if you could give me your opinion on how to proceed
(before I code too much).

eHEA driver writers wants to know what the memory layout is - where the
holes are and where the reserved memory is. They can get this from
parsing through the device-tree every time, but it would be too
expensive. And also, they would like to get information for contiguous
ranges (instead of 16MB chunks).

Since we already have this information at boot time in lmb.memory, 
lmb.reserve - would it be okay to update those for hotplug mem
add/remove ? Of course, all the routines which handles updates 
(lmb_add() etc..) are available only at boot (__init). I could
change that. Is it acceptable ?

Other way to handle the issue is to come up with arch-neutral
way of representing the memory layout. x86-64 and ia64 shows
this information in /proc/iomem (even there is in chunks not 
combined).

Ideas ? Would it be acceptable if I update lmb.memory for add/remove
memory ? Am I missing something ?

Thanks,
Badari

Geoff Levand wrote:

I'm wondering how the memory hot un-plug is initiated on the pseries.
Could you tell me about this HMC?  Is it an application running in
the lpar, or is it an external entity?

The HMC (Hardware Management Console) is a system separate from the
pseries box.  It's used to provision and, um, manage, one or more
systems and their partitions.  It communicates with both the POWER
hypervisor and some optional userspace daemons and utilities running
on the lpars.

Is there a 'standard' interface from userspace that can be used to
trigger the hot-unplug sequence?  I'm asking because PS3's lv1
hypervisor supports hot un-plug of memory, but it would need to be
triggered from some kind of management application running in in
userspace.

I think sysfs is the conventional interface for "offlining" a
resource, that is, getting Linux to stop using it.  That's what is
used for cpu online/offline (and memory as well, I think).  Releasing
a resource to the hypervisor's control is necessarily a
platform-specific operation; on pseries a userspace utility calls a
set of RTAS methods to accomplish this.

On Fri, 2008-02-29 at 09:56 -0800, Badari Pulavarty wrote:

Hi,

I am wondering if you could give me your opinion on how to proceed
(before I code too much).

eHEA driver writers wants to know what the memory layout is - where the
holes are and where the reserved memory is. They can get this from
parsing through the device-tree every time, but it would be too
expensive. And also, they would like to get information for contiguous
ranges (instead of 16MB chunks).

Since we already have this information at boot time in lmb.memory, 
lmb.reserve - would it be okay to update those for hotplug mem
add/remove ? Of course, all the routines which handles updates 
(lmb_add() etc..) are available only at boot (__init). I could
change that. Is it acceptable ?

Other way to handle the issue is to come up with arch-neutral
way of representing the memory layout. x86-64 and ia64 shows
this information in /proc/iomem (even there is in chunks not 
combined).

Ideas ? Would it be acceptable if I update lmb.memory for add/remove
memory ? Am I missing something ?

Here is what I cooked up and it seems to work fine.
It may be easier to review the code (against my earlier patches).

Thanks,
Badari

ppc kernel maintains information about logical memory blocks in
lmb.memory structure at the boot time. Its not updated for
hotplug memory add/remove. hotplug memory notifier for memory
add/remove now updates lmb.memory.

This information is useful for eHEA driver to find out the memory 
layout and holes.

TODO: locking ?

---
 arch/powerpc/mm/lmb.c                           |   61 +++++++++++++++++++++---
 arch/powerpc/platforms/pseries/hotplug-memory.c |   42 ++++++++++++++++
 include/asm-powerpc/lmb.h                       |    3 -
 3 files changed, 98 insertions(+), 8 deletions(-)

Index: linux-2.6.25-rc2/arch/powerpc/mm/lmb.c
===================================================================

--- linux-2.6.25-rc2.orig/arch/powerpc/mm/lmb.c	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/arch/powerpc/mm/lmb.c	2008-02-29 16:55:49.000000000 -0800

@@ -60,13 +60,13 @@ void lmb_dump_all(void)
 #endif /* DEBUG */
 }
 
-static unsigned long __init lmb_addrs_overlap(unsigned long base1,
+static unsigned long lmb_addrs_overlap(unsigned long base1,
 		unsigned long size1, unsigned long base2, unsigned long size2)
 {
 	return ((base1 < (base2+size2)) && (base2 < (base1+size1)));
 }
 
-static long __init lmb_addrs_adjacent(unsigned long base1, unsigned long size1,
+static long lmb_addrs_adjacent(unsigned long base1, unsigned long size1,
 		unsigned long base2, unsigned long size2)
 {
 	if (base2 == base1 + size1)

@@ -77,7 +77,7 @@ static long __init lmb_addrs_adjacent(un
 	return 0;
 }
 
-static long __init lmb_regions_adjacent(struct lmb_region *rgn,
+static long lmb_regions_adjacent(struct lmb_region *rgn,
 		unsigned long r1, unsigned long r2)
 {
 	unsigned long base1 = rgn->region[r1].base;

@@ -88,7 +88,7 @@ static long __init lmb_regions_adjacent(
 	return lmb_addrs_adjacent(base1, size1, base2, size2);
 }
 
-static void __init lmb_remove_region(struct lmb_region *rgn, unsigned long r)
+static void lmb_remove_region(struct lmb_region *rgn, unsigned long r)
 {
 	unsigned long i;
 

@@ -100,7 +100,7 @@ static void __init lmb_remove_region(str
 }
 
 /* Assumption: base addr of region 1 < base addr of region 2 */
-static void __init lmb_coalesce_regions(struct lmb_region *rgn,
+static void lmb_coalesce_regions(struct lmb_region *rgn,
 		unsigned long r1, unsigned long r2)
 {
 	rgn->region[r1].size += rgn->region[r2].size;

@@ -135,7 +135,7 @@ void __init lmb_analyze(void)
 }
 
 /* This routine called with relocation disabled. */
-static long __init lmb_add_region(struct lmb_region *rgn, unsigned long base,
+static long lmb_add_region(struct lmb_region *rgn, unsigned long base,
 				  unsigned long size)
 {
 	unsigned long coalesced = 0;

@@ -191,7 +191,7 @@ static long __init lmb_add_region(struct
 }
 
 /* This routine may be called with relocation disabled. */
-long __init lmb_add(unsigned long base, unsigned long size)
+long lmb_add(unsigned long base, unsigned long size)
 {
 	struct lmb_region *_rgn = &(lmb.memory);
 

@@ -203,6 +203,53 @@ long __init lmb_add(unsigned long base, 
 
 }
 
+long lmb_remove(unsigned long base, unsigned long size)
+{
+	struct lmb_region *rgn = &(lmb.memory);
+	unsigned long end = base + size;
+	unsigned long rgnbegin, rgnend;
+	int i;
+
+	/* Find the region where (base, size) belongs to */
+	for (i=0; i < rgn->cnt; i++) {
+		rgnbegin = rgn->region[i].base;
+		rgnend = rgnbegin + rgn->region[i].size;
+
+		if ((rgnbegin <= base) && (end <= rgnend))
+			break;
+	}
+
+	/* Didn't find the region */
+	if (i == rgn->cnt)
+		return -1;
+
+	/* Check to see if we are removing entire region */
+	if ((rgnbegin == base) && (rgnend == end)) {
+		lmb_remove_region(rgn, i);
+		return 0;
+	}
+
+	/* Check to see if region is matching at the front */
+	if (rgnbegin == base) {
+		rgn->region[i].base = end;
+		rgn->region[i].size -= size;
+		return 0;
+	}
+
+	/* Check to see if the region is matching at the end */
+	if (rgnend == end) {
+		rgn->region[i].size -= size;
+		return 0;
+	}
+
+	/*
+	 * We need to split the entry -  adjust the current one to the
+	 * beginging of the hole and add the region after hole.
+	 */
+	rgn->region[i].size = base - rgn->region[i].base;
+	return lmb_add_region(rgn, end, rgnend - end);
+}
+
 long __init lmb_reserve(unsigned long base, unsigned long size)
 {
 	struct lmb_region *_rgn = &(lmb.reserved);

Index: linux-2.6.25-rc2/include/asm-powerpc/lmb.h
===================================================================

--- linux-2.6.25-rc2.orig/include/asm-powerpc/lmb.h	2008-02-15 12:57:20.000000000 -0800
+++ linux-2.6.25-rc2/include/asm-powerpc/lmb.h	2008-02-29 13:42:03.000000000 -0800

@@ -41,7 +41,8 @@ extern struct lmb lmb;
 
 extern void __init lmb_init(void);
 extern void __init lmb_analyze(void);
-extern long __init lmb_add(unsigned long base, unsigned long size);
+extern long lmb_add(unsigned long base, unsigned long size);
+extern long lmb_remove(unsigned long base, unsigned long size);
 extern long __init lmb_reserve(unsigned long base, unsigned long size);
 extern unsigned long __init lmb_alloc(unsigned long size, unsigned long align);
 extern unsigned long __init lmb_alloc_base(unsigned long size,

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c
===================================================================

--- linux-2.6.25-rc2.orig/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-29 09:25:14.000000000 -0800
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-29 16:58:33.000000000 -0800

@@ -12,6 +12,7 @@
 #include <asm/prom.h>
 #include <asm/firmware.h>
 #include <asm/machdep.h>
+#include <asm/lmb.h>
 #include <asm/pSeries_reconfig.h>
 
 static int pseries_remove_memory(struct device_node *np)

@@ -58,6 +59,11 @@ static int pseries_remove_memory(struct 
 		return ret;
 
 	/*
+	 * Update memory regions for memory remove
+	 */
+	lmb_remove(start_pfn << PAGE_SHIFT, regs[3]);
+
+	/*
 	 * Remove htab bloted mappings for this section of memory
 	 */
  	start = (unsigned long)__va(start_pfn << PAGE_SHIFT);

@@ -65,6 +71,40 @@ static int pseries_remove_memory(struct 
 	return ret;
 }
 
+static int pseries_add_memory(struct device_node *np)
+{
+	const char *type;
+	const unsigned int *my_index;
+	const unsigned int *regs;
+	u64 start_pfn;
+	int ret = -EINVAL;
+
+	/*
+	 * Check to see if we are actually adding memory
+	 */
+	type = of_get_property(np, "device_type", NULL);
+	if (type == NULL || strcmp(type, "memory") != 0)
+		return 0;
+
+	/*
+	 * Find the memory index and size of the removing section
+	 */
+	my_index = of_get_property(np, "ibm,my-drc-index", NULL);
+	if (!my_index)
+		return ret;
+
+	regs = of_get_property(np, "reg", NULL);
+	if (!regs)
+		return ret;
+
+	start_pfn = section_nr_to_pfn(*my_index & 0xffff);
+
+	/*
+	 * Update memory region to represent the memory add
+	 */
+	return lmb_add(start_pfn << PAGE_SHIFT, regs[3]);
+}
+
 static int pseries_memory_notifier(struct notifier_block *nb,
 				unsigned long action, void *node)
 {

@@ -72,6 +112,8 @@ static int pseries_memory_notifier(struc
 
 	switch (action) {
 	case PSERIES_RECONFIG_ADD:
+		if (pseries_add_memory(node))
+			err = NOTIFY_BAD;
 		break;
 	case PSERIES_RECONFIG_REMOVE:
 		if (pseries_remove_memory(node))

From: Badari Pulavarty <redacted>
Date: Fri, 29 Feb 2008 17:12:44 -0800

Here is what I cooked up and it seems to work fine.
It may be easier to review the code (against my earlier patches).

BTW, the lmb code now lives under lib/ (sparc64 will be using it too)
and has several bug fixes applied to it in Paulus's current upstream
tree.

On Fri, 29 Feb 2008 09:47:16 -0800 Badari Pulavarty [off-list ref] wrote:

Here is the latest for review. (just to make sure I didn't miss
anything).

Sorry I didn't review the previous ones ... just a small thing.

+	type = of_get_property(np, "device_type", NULL);

To use the of_ accessors, you need to include linux/of.h (currently
asm/prom.h include linux/of.h, but that may not always be true).  In
fact, I think you don't even need asm/prom.h.

-- 
Cheers,
Stephen Rothwell                    sfr@canb.auug.org.au
http://www.canb.auug.org.au/~sfr/

Stephen Rothwell wrote:

On Fri, 29 Feb 2008 09:47:16 -0800 Badari Pulavarty [off-list ref] wrote:
  

quoted

Here is the latest for review. (just to make sure I didn't miss
anything).
    

Sorry I didn't review the previous ones ... just a small thing.

  

quoted

+	type = of_get_property(np, "device_type", NULL);
    

To use the of_ accessors, you need to include linux/of.h (currently
asm/prom.h include linux/of.h, but that may not always be true).  In
fact, I think you don't even need asm/prom.h.

  

Yes. linux/of.h is enough. Thank you for pointing it out.

Thanks,
Badari

David Miller wrote:

From: Badari Pulavarty <redacted>
Date: Fri, 29 Feb 2008 17:12:44 -0800

  

quoted

Here is what I cooked up and it seems to work fine.
It may be easier to review the code (against my earlier patches).
    

BTW, the lmb code now lives under lib/ (sparc64 will be using it too)
and has several bug fixes applied to it in Paulus's current upstream
tree.
  

Yes. I noticed that earlier. But it was easy for me to test against 
mainline/-mm.
I can easily update it against Paul's tree, if every one is okay with 
the changes
I am proposing.

Thanks,
Badari

On Fri, 2008-02-29 at 09:47 -0800, Badari Pulavarty wrote:

+
+static struct notifier_block pseries_mem_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_mem_nb);
+
+	return 0;
+}
+machine_device_initcall(pseries, pseries_memory_hotplug_init);

Yep that looks good.

cheers

-- 
Michael Ellerman
OzLabs, IBM Australia Development Lab

wwweb: http://michael.ellerman.id.au
phone: +61 2 6212 1183 (tie line 70 21183)

We do not inherit the earth from our ancestors,
we borrow it from our children. - S.M.A.R.T Person

Michael Ellerman wrote:

On Thu, 2008-02-28 at 08:46 -0800, Badari Pulavarty wrote:

quoted

Hotplug memory notifier for ppc64. This gets invoked by writing
the device-node that needs to be removed to /proc/ppc64/ofdt.
We need to adjust the sections and remove sysfs entries by
calling __remove_pages(). Then call arch specific code to
get rid of htab mappings for the section of memory.

Signed-off-by: Badari Pulavarty <redacted>
---
 arch/powerpc/platforms/pseries/Makefile         |    1 
 arch/powerpc/platforms/pseries/hotplug-memory.c |   98 ++++++++++++++++++++++++
 2 files changed, 99 insertions(+)

Index: linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c
===================================================================

--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.25-rc2/arch/powerpc/platforms/pseries/hotplug-memory.c	2008-02-28 08:20:14.000000000 -0800

quoted

+
+static struct notifier_block pseries_smp_nb = {
+	.notifier_call = pseries_memory_notifier,
+};
+
+static int __init pseries_memory_hotplug_init(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		pSeries_reconfig_notifier_register(&pseries_smp_nb);
+
+	return 0;
+}
+arch_initcall(pseries_memory_hotplug_init);

This is going to fire on non-pseries LPAR platforms, like iSeries and
PS3. Which is not what you want I think.

Well, the notifier will be registered, yes, but it will never be
called because that path is reachable only from a write to
/proc/ppc64/ofdt, which is not created on non-pseries.

Maybe it should be

machine_device_initcall(pseries, pseries_memory_hotplug_init);

(and pseries_cpu_hotplug_init in hotplug-cpu.c should be changed to
machine_arch_initcall)