[PATCH v2 0/6] Speedup mremap on ppc64

This patchset enables MOVE_PMD/MOVE_PUD support on power. This requires
the platform to support updating higher-level page tables without
updating page table entries. This also needs to invalidate the Page Walk
Cache on architecture supporting the same.

The patchset does that by switching the page table update to use mmu gather
interface instead of direct tlb flush. mmu gather allows the architecture
to manage page walk cache invalidate separately.

Changes from V1:
* Rebase to recent upstream
* Fix build issues with tlb_gather_mmu changes


Aneesh Kumar K.V (6):
  selftest/mremap_test: Update the test to handle pagesize other than 4K
  selftest/mremap_test: Avoid crash with static build
  mm/mremap: Use pmd/pud_poplulate to update page table entries
  mm/mremap: Use mmu gather interface instead of flush_tlb_range
  mm/mremap: Allow arch runtime override
  powerpc/mm: Enable move pmd/pud

 arch/arc/include/asm/tlb.h               |   5 +
 arch/arm64/include/asm/tlb.h             |   6 ++
 arch/powerpc/include/asm/tlb.h           |   6 ++
 arch/powerpc/platforms/Kconfig.cputype   |   2 +
 arch/x86/include/asm/tlb.h               |   5 +
 mm/mremap.c                              |  54 +++++++++--
 tools/testing/selftests/vm/mremap_test.c | 118 ++++++++++++-----------
 7 files changed, 133 insertions(+), 63 deletions(-)

-- 
2.29.2

Instead of hardcoding 4K page size fetch it using sysconf(). For the performance
measurements test still assume 2M and 1G are hugepage sizes.

Signed-off-by: Aneesh Kumar K.V <redacted>
---
 tools/testing/selftests/vm/mremap_test.c | 113 ++++++++++++-----------
 1 file changed, 61 insertions(+), 52 deletions(-)

diff --git a/tools/testing/selftests/vm/mremap_test.c b/tools/testing/selftests/vm/mremap_test.c
index 9c391d016922..c9a5461eb786 100644
--- a/tools/testing/selftests/vm/mremap_test.c
+++ b/tools/testing/selftests/vm/mremap_test.c

@@ -45,14 +45,15 @@ enum {
 	_4MB = 4ULL << 20,
 	_1GB = 1ULL << 30,
 	_2GB = 2ULL << 30,
-	PTE = _4KB,
 	PMD = _2MB,
 	PUD = _1GB,
 };
 
+#define PTE page_size
+
 #define MAKE_TEST(source_align, destination_align, size,	\
 		  overlaps, should_fail, test_name)		\
-{								\
+(struct test){							\
 	.name = test_name,					\
 	.config = {						\
 		.src_alignment = source_align,			\

@@ -252,12 +253,17 @@ static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
 	return 0;
 }
 
+#define MAX_TEST 13
+#define MAX_PERF_TEST 3
 int main(int argc, char **argv)
 {
 	int failures = 0;
 	int i, run_perf_tests;
 	unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
 	unsigned int pattern_seed;
+	struct test test_cases[MAX_TEST];
+	struct test perf_test_cases[MAX_PERF_TEST];
+	int page_size;
 	time_t t;
 
 	pattern_seed = (unsigned int) time(&t);

@@ -268,56 +274,59 @@ int main(int argc, char **argv)
 	ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
 		       threshold_mb, pattern_seed);
 
-	struct test test_cases[] = {
-		/* Expected mremap failures */
-		MAKE_TEST(_4KB, _4KB, _4KB, OVERLAPPING, EXPECT_FAILURE,
-		  "mremap - Source and Destination Regions Overlapping"),
-		MAKE_TEST(_4KB, _1KB, _4KB, NON_OVERLAPPING, EXPECT_FAILURE,
-		  "mremap - Destination Address Misaligned (1KB-aligned)"),
-		MAKE_TEST(_1KB, _4KB, _4KB, NON_OVERLAPPING, EXPECT_FAILURE,
-		  "mremap - Source Address Misaligned (1KB-aligned)"),
-
-		/* Src addr PTE aligned */
-		MAKE_TEST(PTE, PTE, _8KB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "8KB mremap - Source PTE-aligned, Destination PTE-aligned"),
-
-		/* Src addr 1MB aligned */
-		MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"),
-		MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"),
-
-		/* Src addr PMD aligned */
-		MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "4MB mremap - Source PMD-aligned, Destination PTE-aligned"),
-		MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"),
-		MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "4MB mremap - Source PMD-aligned, Destination PMD-aligned"),
-
-		/* Src addr PUD aligned */
-		MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "2GB mremap - Source PUD-aligned, Destination PTE-aligned"),
-		MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"),
-		MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "2GB mremap - Source PUD-aligned, Destination PMD-aligned"),
-		MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "2GB mremap - Source PUD-aligned, Destination PUD-aligned"),
-	};
-
-	struct test perf_test_cases[] = {
-		/*
-		 * mremap 1GB region - Page table level aligned time
-		 * comparison.
-		 */
-		MAKE_TEST(PTE, PTE, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "1GB mremap - Source PTE-aligned, Destination PTE-aligned"),
-		MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "1GB mremap - Source PMD-aligned, Destination PMD-aligned"),
-		MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
-		  "1GB mremap - Source PUD-aligned, Destination PUD-aligned"),
-	};
+	page_size = sysconf(_SC_PAGESIZE);
+
+	/* Expected mremap failures */
+	test_cases[0] =	MAKE_TEST(page_size, page_size, page_size,
+				  OVERLAPPING, EXPECT_FAILURE,
+				  "mremap - Source and Destination Regions Overlapping");
+
+	test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
+				  NON_OVERLAPPING, EXPECT_FAILURE,
+				  "mremap - Destination Address Misaligned (1KB-aligned)");
+	test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
+				  NON_OVERLAPPING, EXPECT_FAILURE,
+				  "mremap - Source Address Misaligned (1KB-aligned)");
+
+	/* Src addr PTE aligned */
+	test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
+				  NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
+
+	/* Src addr 1MB aligned */
+	test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
+	test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
+
+	/* Src addr PMD aligned */
+	test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
+	test_cases[7] =	MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
+	test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
+
+	/* Src addr PUD aligned */
+	test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				  "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
+	test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				   "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
+	test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				   "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
+	test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				   "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
+
+	perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+					"1GB mremap - Source PTE-aligned, Destination PTE-aligned");
+	/*
+	 * mremap 1GB region - Page table level aligned time
+	 * comparison.
+	 */
+	perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				       "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
+	perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
+				       "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
 
 	run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
 				(threshold_mb * _1MB >= _1GB);

-- 
2.29.2

pmd/pud_populate is the right interface to be used to set the respective
page table entries. Some architectures like ppc64 do assume that set_pmd/pud_at
can only be used to set a hugepage PTE. Since we are not setting up a hugepage
PTE here, use the pmd/pud_populate interface.

Signed-off-by: Aneesh Kumar K.V <redacted>
---
 mm/mremap.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/mm/mremap.c b/mm/mremap.c
index ec8f840399ed..574287f9bb39 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c

@@ -26,6 +26,7 @@
 
 #include <asm/cacheflush.h>
 #include <asm/tlbflush.h>
+#include <asm/pgalloc.h>
 
 #include "internal.h"
 

@@ -257,9 +258,8 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 	pmd_clear(old_pmd);
 
 	VM_BUG_ON(!pmd_none(*new_pmd));
+	pmd_populate(mm, new_pmd, (pgtable_t)pmd_page_vaddr(pmd));
 
-	/* Set the new pmd */
-	set_pmd_at(mm, new_addr, new_pmd, pmd);
 	flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
 	if (new_ptl != old_ptl)
 		spin_unlock(new_ptl);

@@ -306,8 +306,7 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
 
 	VM_BUG_ON(!pud_none(*new_pud));
 
-	/* Set the new pud */
-	set_pud_at(mm, new_addr, new_pud, pud);
+	pud_populate(mm, new_pud, (pmd_t *)pud_page_vaddr(pud));
 	flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE);
 	if (new_ptl != old_ptl)
 		spin_unlock(new_ptl);

-- 
2.29.2

Architectures like ppc64 can only support faster mremap only with radix
translation. Hence allow a runtime check w.r.t support for fast mremap.

Signed-off-by: Aneesh Kumar K.V <redacted>
---
 arch/arc/include/asm/tlb.h     |  5 +++++
 arch/arm64/include/asm/tlb.h   |  6 ++++++
 arch/powerpc/include/asm/tlb.h |  6 ++++++
 arch/x86/include/asm/tlb.h     |  5 +++++
 mm/mremap.c                    | 14 +++++++++++++-
 5 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/arch/arc/include/asm/tlb.h b/arch/arc/include/asm/tlb.h
index 975b35d3738d..22b8cfb46cbf 100644
--- a/arch/arc/include/asm/tlb.h
+++ b/arch/arc/include/asm/tlb.h

@@ -9,4 +9,9 @@
 #include <linux/pagemap.h>
 #include <asm-generic/tlb.h>
 
+#define arch_supports_page_tables_move arch_supports_page_tables_move
+static inline bool arch_supports_page_tables_move(void)
+{
+	return true;
+}
 #endif /* _ASM_ARC_TLB_H */

diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 61c97d3b58c7..fe209efc6a10 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h

@@ -94,4 +94,10 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
 }
 #endif
 
+#define arch_supports_page_tables_move arch_supports_page_tables_move
+static inline bool arch_supports_page_tables_move(void)
+{
+	return true;
+}
+
 #endif

diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h
index 160422a439aa..058918a7cd3c 100644
--- a/arch/powerpc/include/asm/tlb.h
+++ b/arch/powerpc/include/asm/tlb.h

@@ -83,5 +83,11 @@ static inline int mm_is_thread_local(struct mm_struct *mm)
 }
 #endif
 
+#define arch_supports_page_tables_move arch_supports_page_tables_move
+static inline bool arch_supports_page_tables_move(void)
+{
+	return radix_enabled();
+}
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_POWERPC_TLB_H */

diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 1bfe979bb9bc..62915238bb36 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h

@@ -37,4 +37,9 @@ static inline void __tlb_remove_table(void *table)
 	free_page_and_swap_cache(table);
 }
 
+#define arch_supports_page_tables_move arch_supports_page_tables_move
+static inline bool arch_supports_page_tables_move(void)
+{
+	return true;
+}
 #endif /* _ASM_X86_TLB_H */

diff --git a/mm/mremap.c b/mm/mremap.c
index fafa73b965d3..316181822cce 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c

@@ -25,7 +25,7 @@
 #include <linux/userfaultfd_k.h>
 
 #include <asm/cacheflush.h>
-#include <asm/tlbflush.h>
+#include <asm/tlb.h>
 #include <asm/pgalloc.h>
 
 #include "internal.h"

@@ -210,6 +210,14 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
 		drop_rmap_locks(vma);
 }
 
+#ifndef arch_supports_page_tables_move
+#define arch_supports_page_tables_move arch_supports_page_tables_move
+static inline bool arch_supports_page_tables_move(void)
+{
+	return false;
+}
+#endif
+
 #ifdef CONFIG_HAVE_MOVE_PMD
 static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 		  unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)

@@ -219,6 +227,8 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 	struct mmu_gather tlb;
 	pmd_t pmd;
 
+	if (!arch_supports_page_tables_move())
+		return false;
 	/*
 	 * The destination pmd shouldn't be established, free_pgtables()
 	 * should have released it.

@@ -297,6 +307,8 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
 	struct mmu_gather tlb;
 	pud_t pud;
 
+	if (!arch_supports_page_tables_move())
+		return false;
 	/*
 	 * The destination pud shouldn't be established, free_pgtables()
 	 * should have released it.

-- 
2.29.2

Some architectures do have the concept of page walk cache and only mmu gather
interface supports flushing them. A fast mremap that involves moving page
table pages instead of copying pte entries should flush page walk cache since
the old translation cache is no more valid. Hence switch to mm gather to flush
TLB and mark tlb.freed_tables = 1. No page table pages need to be freed here.
With this the tlb flush is done outside page table lock (ptl).

Signed-off-by: Aneesh Kumar K.V <redacted>
---
 mm/mremap.c | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/mm/mremap.c b/mm/mremap.c
index 574287f9bb39..fafa73b965d3 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c

@@ -216,6 +216,7 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 {
 	spinlock_t *old_ptl, *new_ptl;
 	struct mm_struct *mm = vma->vm_mm;
+	struct mmu_gather tlb;
 	pmd_t pmd;
 
 	/*

@@ -244,11 +245,12 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 	if (WARN_ON_ONCE(!pmd_none(*new_pmd)))
 		return false;
 
+	tlb_gather_mmu(&tlb, mm);
 	/*
 	 * We don't have to worry about the ordering of src and dst
 	 * ptlocks because exclusive mmap_lock prevents deadlock.
 	 */
-	old_ptl = pmd_lock(vma->vm_mm, old_pmd);
+	old_ptl = pmd_lock(mm, old_pmd);
 	new_ptl = pmd_lockptr(mm, new_pmd);
 	if (new_ptl != old_ptl)
 		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);

@@ -257,13 +259,23 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 	pmd = *old_pmd;
 	pmd_clear(old_pmd);
 
+	/*
+	 * Mark the range. We are not freeing page table pages nor
+	 * regular pages. Hence we don't need to call tlb_remove_table()
+	 * or tlb_remove_page().
+	 */
+	tlb_flush_pte_range(&tlb, old_addr, PMD_SIZE);
+	tlb.freed_tables = 1;
 	VM_BUG_ON(!pmd_none(*new_pmd));
 	pmd_populate(mm, new_pmd, (pgtable_t)pmd_page_vaddr(pmd));
 
-	flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
 	if (new_ptl != old_ptl)
 		spin_unlock(new_ptl);
 	spin_unlock(old_ptl);
+	/*
+	 * This will invalidate both the old TLB and page table walk caches.
+	 */
+	tlb_finish_mmu(&tlb);
 
 	return true;
 }

@@ -282,6 +294,7 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
 {
 	spinlock_t *old_ptl, *new_ptl;
 	struct mm_struct *mm = vma->vm_mm;
+	struct mmu_gather tlb;
 	pud_t pud;
 
 	/*

@@ -291,11 +304,12 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
 	if (WARN_ON_ONCE(!pud_none(*new_pud)))
 		return false;
 
+	tlb_gather_mmu(&tlb, mm);
 	/*
 	 * We don't have to worry about the ordering of src and dst
 	 * ptlocks because exclusive mmap_lock prevents deadlock.
 	 */
-	old_ptl = pud_lock(vma->vm_mm, old_pud);
+	old_ptl = pud_lock(mm, old_pud);
 	new_ptl = pud_lockptr(mm, new_pud);
 	if (new_ptl != old_ptl)
 		spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);

@@ -304,14 +318,25 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
 	pud = *old_pud;
 	pud_clear(old_pud);
 
+	/*
+	 * Mark the range. We are not freeing page table pages nor
+	 * regular pages. Hence we don't need to call tlb_remove_table()
+	 * or tlb_remove_page().
+	 */
+	tlb_flush_pte_range(&tlb, old_addr, PUD_SIZE);
+	tlb.freed_tables = 1;
 	VM_BUG_ON(!pud_none(*new_pud));
 
 	pud_populate(mm, new_pud, (pmd_t *)pud_page_vaddr(pud));
-	flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE);
+
 	if (new_ptl != old_ptl)
 		spin_unlock(new_ptl);
 	spin_unlock(old_ptl);
 
+	/*
+	 * This will invalidate both the old TLB and page table walk caches.
+	 */
+	tlb_finish_mmu(&tlb);
 	return true;
 }
 #else

-- 
2.29.2

With a large mmap map size, we can overlap with the text area and using
MAP_FIXED results in unmapping that area. Switch to MAP_FIXED_NOREPLACE
and handle the EEXIST error.

Signed-off-by: Aneesh Kumar K.V <redacted>
---
 tools/testing/selftests/vm/mremap_test.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/testing/selftests/vm/mremap_test.c b/tools/testing/selftests/vm/mremap_test.c
index c9a5461eb786..0624d1bd71b5 100644
--- a/tools/testing/selftests/vm/mremap_test.c
+++ b/tools/testing/selftests/vm/mremap_test.c

@@ -75,9 +75,10 @@ static void *get_source_mapping(struct config c)
 retry:
 	addr += c.src_alignment;
 	src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
-			MAP_FIXED | MAP_ANONYMOUS | MAP_SHARED, -1, 0);
+			MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
+			-1, 0);
 	if (src_addr == MAP_FAILED) {
-		if (errno == EPERM)
+		if (errno == EPERM || errno == EEXIST)
 			goto retry;
 		goto error;
 	}

-- 
2.29.2

mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:
1GB mremap - Source PTE-aligned, Destination PTE-aligned
        mremap time:      1114318ns
1GB mremap - Source PMD-aligned, Destination PMD-aligned
        mremap time:      1097715ns
1GB mremap - Source PUD-aligned, Destination PUD-aligned
        mremap time:        26851ns

Signed-off-by: Aneesh Kumar K.V <redacted>
---
 arch/powerpc/platforms/Kconfig.cputype | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 3ce907523b1e..2e666e569fdf 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype

@@ -97,6 +97,8 @@ config PPC_BOOK3S_64
 	select PPC_HAVE_PMU_SUPPORT
 	select SYS_SUPPORTS_HUGETLBFS
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
+	select HAVE_MOVE_PMD
+	select HAVE_MOVE_PUD
 	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select IRQ_WORK

-- 
2.29.2

Hi "Aneesh,

I love your patch! Yet something to improve:

[auto build test ERROR on powerpc/next]
[also build test ERROR on kselftest/next v5.12-rc3 next-20210317]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Aneesh-Kumar-K-V/Speedup-mremap-on-ppc64/20210315-194324
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: x86_64-rhel-8.3 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
        # https://github.com/0day-ci/linux/commit/d3b9a3e6f414413d8f822185158b937d9f19b7a6
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Aneesh-Kumar-K-V/Speedup-mremap-on-ppc64/20210315-194324
        git checkout d3b9a3e6f414413d8f822185158b937d9f19b7a6
        # save the attached .config to linux build tree
        make W=1 ARCH=x86_64 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <redacted>

Note: the linux-review/Aneesh-Kumar-K-V/Speedup-mremap-on-ppc64/20210315-194324 HEAD 79633714ff2b990b3e4972873457678bb34d029f builds fine.
      It only hurts bisectibility.

All errors (new ones prefixed by >>):

   mm/mremap.c: In function 'move_normal_pmd':

quoted

mm/mremap.c:219:20: error: storage size of 'tlb' isn't known

     219 |  struct mmu_gather tlb;
         |                    ^~~

quoted

mm/mremap.c:267:2: error: implicit declaration of function 'tlb_flush_pte_range' [-Werror=implicit-function-declaration]

     267 |  tlb_flush_pte_range(&tlb, old_addr, PMD_SIZE);
         |  ^~~~~~~~~~~~~~~~~~~
   mm/mremap.c:219:20: warning: unused variable 'tlb' [-Wunused-variable]
     219 |  struct mmu_gather tlb;
         |                    ^~~
   mm/mremap.c: In function 'move_normal_pud':
   mm/mremap.c:297:20: error: storage size of 'tlb' isn't known
     297 |  struct mmu_gather tlb;
         |                    ^~~
   mm/mremap.c:297:20: warning: unused variable 'tlb' [-Wunused-variable]
   cc1: some warnings being treated as errors


vim +219 mm/mremap.c

   212	
   213	#ifdef CONFIG_HAVE_MOVE_PMD
   214	static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
   215			  unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
   216	{
   217		spinlock_t *old_ptl, *new_ptl;
   218		struct mm_struct *mm = vma->vm_mm;
 > 219		struct mmu_gather tlb;
   220		pmd_t pmd;
   221	
   222		/*
   223		 * The destination pmd shouldn't be established, free_pgtables()
   224		 * should have released it.
   225		 *
   226		 * However, there's a case during execve() where we use mremap
   227		 * to move the initial stack, and in that case the target area
   228		 * may overlap the source area (always moving down).
   229		 *
   230		 * If everything is PMD-aligned, that works fine, as moving
   231		 * each pmd down will clear the source pmd. But if we first
   232		 * have a few 4kB-only pages that get moved down, and then
   233		 * hit the "now the rest is PMD-aligned, let's do everything
   234		 * one pmd at a time", we will still have the old (now empty
   235		 * of any 4kB pages, but still there) PMD in the page table
   236		 * tree.
   237		 *
   238		 * Warn on it once - because we really should try to figure
   239		 * out how to do this better - but then say "I won't move
   240		 * this pmd".
   241		 *
   242		 * One alternative might be to just unmap the target pmd at
   243		 * this point, and verify that it really is empty. We'll see.
   244		 */
   245		if (WARN_ON_ONCE(!pmd_none(*new_pmd)))
   246			return false;
   247	
   248		tlb_gather_mmu(&tlb, mm);
   249		/*
   250		 * We don't have to worry about the ordering of src and dst
   251		 * ptlocks because exclusive mmap_lock prevents deadlock.
   252		 */
   253		old_ptl = pmd_lock(mm, old_pmd);
   254		new_ptl = pmd_lockptr(mm, new_pmd);
   255		if (new_ptl != old_ptl)
   256			spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
   257	
   258		/* Clear the pmd */
   259		pmd = *old_pmd;
   260		pmd_clear(old_pmd);
   261	
   262		/*
   263		 * Mark the range. We are not freeing page table pages nor
   264		 * regular pages. Hence we don't need to call tlb_remove_table()
   265		 * or tlb_remove_page().
   266		 */
 > 267		tlb_flush_pte_range(&tlb, old_addr, PMD_SIZE);
   268		tlb.freed_tables = 1;
   269		VM_BUG_ON(!pmd_none(*new_pmd));
   270		pmd_populate(mm, new_pmd, (pgtable_t)pmd_page_vaddr(pmd));
   271	
   272		if (new_ptl != old_ptl)
   273			spin_unlock(new_ptl);
   274		spin_unlock(old_ptl);
   275		/*
   276		 * This will invalidate both the old TLB and page table walk caches.
   277		 */
   278		tlb_finish_mmu(&tlb);
   279	
   280		return true;
   281	}
   282	#else
   283	static inline bool move_normal_pmd(struct vm_area_struct *vma,
   284			unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd,
   285			pmd_t *new_pmd)
   286	{
   287		return false;
   288	}
   289	#endif
   290	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

Excerpts from Aneesh Kumar K.V's message of March 15, 2021 9:38 pm:

Some architectures do have the concept of page walk cache and only mmu gather
interface supports flushing them. A fast mremap that involves moving page
table pages instead of copying pte entries should flush page walk cache since
the old translation cache is no more valid. Hence switch to mm gather to flush
TLB and mark tlb.freed_tables = 1. No page table pages need to be freed here.
With this the tlb flush is done outside page table lock (ptl).

I would maybe just get archs that implement it to provide a specific
flush_tlb+pwc_range for it, or else they get flush_tlb_range by default.

I think that would be simpler for now, at least in generic code.

There was some other talk of consolidating the TLB flush APIs, I jsut 
don't know if it's the best way to go to use the page/page table 
gathering and freeing API for it.

Thanks,
Nick

Greeting,

FYI, we noticed the following commit (built with gcc-9):

commit: c10e9d3441ddee43a36482980f1e39efbd5169e2 ("[PATCH v2 3/6] mm/mremap: Use pmd/pud_poplulate to update page table entries")
url: https://github.com/0day-ci/linux/commits/Aneesh-Kumar-K-V/Speedup-mremap-on-ppc64/20210315-194324
base: https://git.kernel.org/cgit/linux/kernel/git/powerpc/linux.git next

in testcase: ltp
version: ltp-x86_64-14c1f76-1_20210318
with following parameters:

	test: cve
	ucode: 0xe2

test-description: The LTP testsuite contains a collection of tools for testing the Linux kernel and related features.
test-url: http://linux-test-project.github.io/


on test machine: 8 threads Intel(R) Core(TM) i7-6770HQ CPU @ 2.60GHz with 32G memory

caused below changes (please refer to attached dmesg/kmsg for entire log/backtrace):



If you fix the issue, kindly add following tag
Reported-by: kernel test robot <redacted>


[   98.252964] INFO: creating /lkp/benchmarks/ltp/output directory
[   98.252970]
[   98.267491] INFO: creating /lkp/benchmarks/ltp/results directory
[   98.267497]
[   98.281384] Checking for required user/group ids
[   98.281391]
[   98.291431]
[   98.291435]
[   98.300185] 'nobody' user id and group found.
[   98.300191]
[   98.311932] 'bin' user id and group found.
[   98.311939]
[   98.323511] 'daemon' user id and group found.
[   98.323518]
[   98.335042] Users group found.
[   98.335048]
[   98.344952] Sys group found.
[   98.344959]
[   98.355096] Required users/groups exist.
[   98.355102]
[   98.368977] If some fields are empty or look unusual you may have an old version.
[   98.368985]
[   98.386687] Compare to the current minimal requirements in Documentation/Changes.
[   98.386695]
[   98.400203]
[   98.400208]
[   98.408018] /etc/os-release
[   98.408024]
[   98.417964] PRETTY_NAME="Debian GNU/Linux 10 (buster)"
[   98.417970]
[   98.429645] NAME="Debian GNU/Linux"
[   98.429651]
[   98.439140] VERSION_ID="10"
[   98.439146]
[   98.448259] VERSION="10 (buster)"
[   98.448264]
[   98.457722] VERSION_CODENAME=buster
[   98.457727]
[   98.466720] ID=debian
[   98.466724]
[   98.475508] HOME_URL="https://www.debian.org/"
[   98.475513]
[   98.486989] SUPPORT_URL="https://www.debian.org/support"
[   98.486994]
[   98.499083] BUG_REPORT_URL="https://bugs.debian.org/"
[   98.499088]
[   98.508857]
[   98.508861]
[   98.515024] uname:
[   98.515029]
[   98.526128] Linux lkp-skl-nuc2 5.12.0-rc2-00014-gc10e9d3441dd #1 SMP Thu Mar 18 13:26:11 CST 2021 x86_64 GNU/Linux
[   98.526134]
[   98.541084]
[   98.541088]
[   98.547140] /proc/cmdline
[   98.547145]
[  110.939655] ip=::::lkp-skl-nuc2::dhcp root=/dev/ram0 user=lkp job=/lkp/jobs/scheduled/lkp-skl-nuc2/ltp-cve-ucode=0xe2-debian-10.4-x86_64-20200603.cgz-c10e9d3441ddee43a36482980f1e39efbd5169e2-20210319-46118-1o3779n-5.yaml ARCH=x86_64 kconfig=x86_64-rhel-8.3 branch=linux-review/Aneesh-Kumar-K-V/Speedup-mremap-on-ppc64/20210315-194324 commit=c10e9d3441ddee43a36482980f1e39efbd5169e2 BOOT_IMAGE=/pkg/linux/x86_64-rhel-8.3/gcc-9/c10e9d3441ddee43a36482980f1e39efbd5169e2/vmlinuz-5.12.0-rc2-00014-gc10e9d3441dd max_uptime=2100 RESULT_ROOT=/result/ltp/cve-ucode=0xe2/lkp-skl-nuc2/debian-10.4-x86_64-20200603.cgz/x86_64-rhel-8.3/gcc-9/c10e9d3441ddee43a36482980f1e39efbd5169e2/1 LKP_SERVER=internal-lkp-server nokaslr selinux=0 debug apic=debug sysrq_always_enabled rcupdate.rcu_cpu_stall_timeout=100 net.ifnames=0 printk.devkmsg=on panic=-1 softlockup_panic=1 nmi_watchdog=panic oops=panic load_ramdisk=2 prompt_ramdisk
[  110.939673]
[  111.035045]
[  111.035051]
[  111.428862] true: Corrupted page table at address 7ffce3c00000
[  111.436435] PGD 80000008bb481067 P4D 80000008bb481067 PUD 8b23e0067 PMD ffe7a22cdb3c0067
[  111.436442] BAD
[  111.449812] Bad pagetable: 0009 [#1] SMP PTI
[  111.455735] CPU: 2 PID: 3350 Comm: true Tainted: G          I       5.12.0-rc2-00014-gc10e9d3441dd #1
[  111.466839] Hardware name:  /NUC6i7KYB, BIOS KYSKLi70.86A.0041.2016.0817.1130 08/17/2016
[  111.476757] RIP: 0010:strnlen_user (kbuild/src/consumer/lib/strnlen_user.c:52 kbuild/src/consumer/lib/strnlen_user.c:113) 
[ 111.483021] Code: 80 80 4c 09 da 45 31 db 48 8d 0c 1a 48 f7 d2 48 21 ca 4c 21 c2 75 2e 48 89 c1 49 8d 6a 08 4c 29 d1 48 83 f9 08 76 62 44 89 d9 <4a> 8b 54 17 08 85 c9 75 4b 48 8d 0c 1a 48 f7 d2 49 89 ea 48 21 ca
All code
========
   0:	80 80 4c 09 da 45 31 	addb   $0x31,0x45da094c(%rax)
   7:	db 48 8d             	fisttpl -0x73(%rax)
   a:	0c 1a                	or     $0x1a,%al
   c:	48 f7 d2             	not    %rdx
   f:	48 21 ca             	and    %rcx,%rdx
  12:	4c 21 c2             	and    %r8,%rdx
  15:	75 2e                	jne    0x45
  17:	48 89 c1             	mov    %rax,%rcx
  1a:	49 8d 6a 08          	lea    0x8(%r10),%rbp
  1e:	4c 29 d1             	sub    %r10,%rcx
  21:	48 83 f9 08          	cmp    $0x8,%rcx
  25:	76 62                	jbe    0x89
  27:	44 89 d9             	mov    %r11d,%ecx
  2a:*	4a 8b 54 17 08       	mov    0x8(%rdi,%r10,1),%rdx		<-- trapping instruction
  2f:	85 c9                	test   %ecx,%ecx
  31:	75 4b                	jne    0x7e
  33:	48 8d 0c 1a          	lea    (%rdx,%rbx,1),%rcx
  37:	48 f7 d2             	not    %rdx
  3a:	49 89 ea             	mov    %rbp,%r10
  3d:	48 21 ca             	and    %rcx,%rdx

Code starting with the faulting instruction
===========================================
   0:	4a 8b 54 17 08       	mov    0x8(%rdi,%r10,1),%rdx
   5:	85 c9                	test   %ecx,%ecx
   7:	75 4b                	jne    0x54
   9:	48 8d 0c 1a          	lea    (%rdx,%rbx,1),%rcx
   d:	48 f7 d2             	not    %rdx
  10:	49 89 ea             	mov    %rbp,%r10
  13:	48 21 ca             	and    %rcx,%rdx
[  111.505997] RSP: 0018:ffffc90002ecfcb8 EFLAGS: 00050202
[  111.513206] RAX: 0000000000020002 RBX: fefefefefefefeff RCX: 0000000000000000
[  111.522461] RDX: 0000000000000000 RSI: 0000000000020000 RDI: 00007ffce3bffaf0
[  111.531682] RBP: 0000000000000510 R08: 8080808080808080 R09: 0000000000000002
[  111.540826] R10: 0000000000000508 R11: 0000000000000000 R12: 00007ffce3a00d08
[  111.550065] R13: 00007ffce3bffaf2 R14: ffff8888bca2a200 R15: 00000000000005fc
[  111.559310] FS:  0000000000000000(0000) GS:ffff8888bec80000(0000) knlGS:0000000000000000
[  111.569563] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  111.577436] CR2: 00007ffce3c00000 CR3: 00000008b2770001 CR4: 00000000003706e0
[  111.586711] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[  111.596002] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[  111.605227] Call Trace:
[  111.609613] create_elf_tables (kbuild/src/consumer/fs/binfmt_elf.c:331) 
[  111.616334] load_elf_binary (kbuild/src/consumer/fs/binfmt_elf.c:1257) 
[  111.622317] exec_binprm (kbuild/src/consumer/fs/exec.c:1722 kbuild/src/consumer/fs/exec.c:1761) 
[  111.627934] bprm_execve (kbuild/src/consumer/fs/exec.c:1831 kbuild/src/consumer/fs/exec.c:1792) 
[  111.633548] do_execveat_common+0x18f/0x1c0 
[  111.640406] __x64_sys_execve (kbuild/src/consumer/fs/exec.c:2058) 
[  111.646280] do_syscall_64 (kbuild/src/consumer/arch/x86/entry/common.c:46) 
[  111.651934] entry_SYSCALL_64_after_hwframe (kbuild/src/consumer/arch/x86/entry/entry_64.S:112) 
[  111.659119] RIP: 0033:0x7f58cf452a07
[ 111.664728] Code: Unable to access opcode bytes at RIP 0x7f58cf4529dd.

Code starting with the faulting instruction
===========================================


To reproduce:

        git clone https://github.com/intel/lkp-tests.git
        cd lkp-tests
        bin/lkp install                job.yaml  # job file is attached in this email
        bin/lkp split-job --compatible job.yaml
        bin/lkp run                    compatible-job.yaml



---
0DAY/LKP+ Test Infrastructure                   Open Source Technology Center
https://lists.01.org/hyperkitty/list/lkp@lists.01.org       Intel Corporation

Thanks,
Oliver Sang