--- v6
+++ v4
@@ -1,181 +1,80 @@
-No functional change in this patch
+Some architectures do have the concept of page walk cache which need
+to be flush when updating higher levels of page tables. A fast mremap
+that involves moving page table pages instead of copying pte entries
+should flush page walk cache since the old translation cache is no more
+valid.
+
+Add new helper flush_pte_tlb_pwc_range() which invalidates both TLB and
+page walk cache where TLB entries are mapped with page size PAGE_SIZE.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
- .../include/asm/book3s/64/tlbflush-radix.h | 19 +++++++-----
- arch/powerpc/include/asm/book3s/64/tlbflush.h | 23 ++++++++++++---
- arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 4 +--
- arch/powerpc/mm/book3s64/radix_tlb.c | 29 +++++++------------
- 4 files changed, 42 insertions(+), 33 deletions(-)
+ arch/powerpc/include/asm/book3s/64/tlbflush.h | 11 +++++++++++
+ mm/mremap.c | 15 +++++++++++++--
+ 2 files changed, 24 insertions(+), 2 deletions(-)
-diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
-index 8b33601cdb9d..171441a43b35 100644
---- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
-+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
-@@ -56,15 +56,18 @@ static inline void radix__flush_all_lpid_guest(unsigned int lpid)
- }
- #endif
-
--extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
-- unsigned long start, unsigned long end);
--extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
-- unsigned long end, int psize);
--extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
-- unsigned long start, unsigned long end);
--extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
-+void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
-+ unsigned long start, unsigned long end,
-+ bool flush_pwc);
-+void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
-+ unsigned long start, unsigned long end,
-+ bool flush_pwc);
-+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
-+ unsigned long end, int psize, bool flush_pwc);
-+void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
- unsigned long end);
--extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
-+void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
-+
-
- extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
- extern void radix__local_flush_all_mm(struct mm_struct *mm);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
-index 215973b4cb26..f9f8a3a264f7 100644
+index f9f8a3a264f7..c236b66f490b 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
-@@ -45,13 +45,30 @@ static inline void tlbiel_all_lpid(bool radix)
- hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
+@@ -80,6 +80,17 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
+ return flush_hugetlb_tlb_pwc_range(vma, start, end, false);
}
-+static inline void flush_pmd_tlb_pwc_range(struct vm_area_struct *vma,
++#define flush_pte_tlb_pwc_range flush_tlb_pwc_range
++static inline void flush_pte_tlb_pwc_range(struct vm_area_struct *vma,
++ unsigned long start, unsigned long end,
++ bool also_pwc)
++{
++ if (radix_enabled())
++ return radix__flush_tlb_pwc_range_psize(vma->vm_mm, start,
++ end, mmu_virtual_psize, also_pwc);
++ return hash__flush_tlb_range(vma, start, end);
++}
++
+ static inline void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+ {
+diff --git a/mm/mremap.c b/mm/mremap.c
+index 574287f9bb39..0e7b11daafee 100644
+--- a/mm/mremap.c
++++ b/mm/mremap.c
+@@ -210,6 +210,17 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
+ drop_rmap_locks(vma);
+ }
+
++#ifndef flush_pte_tlb_pwc_range
++#define flush_pte_tlb_pwc_range flush_pte_tlb_pwc_range
++static inline void flush_pte_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
-+ bool flush_pwc)
++ bool also_pwc)
+{
-+ if (radix_enabled())
-+ return radix__flush_pmd_tlb_range(vma, start, end, flush_pwc);
-+ return hash__flush_tlb_range(vma, start, end);
++ return flush_tlb_range(vma, start, end);
+}
++#endif
++
+ #ifdef CONFIG_HAVE_MOVE_PMD
+ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
+ unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
+@@ -260,7 +271,7 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr,
+ VM_BUG_ON(!pmd_none(*new_pmd));
+ pmd_populate(mm, new_pmd, (pgtable_t)pmd_page_vaddr(pmd));
- #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
- static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
-+{
-+ return flush_pmd_tlb_pwc_range(vma, start, end, false);
-+}
-+
-+static inline void flush_hugetlb_tlb_pwc_range(struct vm_area_struct *vma,
-+ unsigned long start,
-+ unsigned long end,
-+ bool flush_pwc)
- {
- if (radix_enabled())
-- return radix__flush_pmd_tlb_range(vma, start, end);
-+ return radix__flush_hugetlb_tlb_range(vma, start, end, flush_pwc);
- return hash__flush_tlb_range(vma, start, end);
- }
+- flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
++ flush_pte_tlb_pwc_range(vma, old_addr, old_addr + PMD_SIZE, true);
+ if (new_ptl != old_ptl)
+ spin_unlock(new_ptl);
+ spin_unlock(old_ptl);
+@@ -307,7 +318,7 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr,
+ VM_BUG_ON(!pud_none(*new_pud));
-@@ -60,9 +77,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
- unsigned long start,
- unsigned long end)
- {
-- if (radix_enabled())
-- return radix__flush_hugetlb_tlb_range(vma, start, end);
-- return hash__flush_tlb_range(vma, start, end);
-+ return flush_hugetlb_tlb_pwc_range(vma, start, end, false);
- }
-
- static inline void flush_tlb_range(struct vm_area_struct *vma,
-diff --git a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
-index cb91071eef52..e62f5679b119 100644
---- a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
-+++ b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
-@@ -26,13 +26,13 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v
- }
-
- void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
-- unsigned long end)
-+ unsigned long end, bool flush_pwc)
- {
- int psize;
- struct hstate *hstate = hstate_file(vma->vm_file);
-
- psize = hstate_get_psize(hstate);
-- radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize);
-+ radix__flush_tlb_pwc_range_psize(vma->vm_mm, start, end, psize, flush_pwc);
- }
-
- /*
-diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
-index 817a02ef6032..5a59e19f9e53 100644
---- a/arch/powerpc/mm/book3s64/radix_tlb.c
-+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
-@@ -1090,7 +1090,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
- {
- #ifdef CONFIG_HUGETLB_PAGE
- if (is_vm_hugetlb_page(vma))
-- return radix__flush_hugetlb_tlb_range(vma, start, end);
-+ return radix__flush_hugetlb_tlb_range(vma, start, end, false);
- #endif
-
- __radix__flush_tlb_range(vma->vm_mm, start, end);
-@@ -1151,9 +1151,6 @@ void radix__flush_all_lpid_guest(unsigned int lpid)
- _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
- }
-
--static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
-- unsigned long end, int psize);
--
- void radix__tlb_flush(struct mmu_gather *tlb)
- {
- int psize = 0;
-@@ -1177,10 +1174,8 @@ void radix__tlb_flush(struct mmu_gather *tlb)
- else
- radix__flush_all_mm(mm);
- } else {
-- if (!tlb->freed_tables)
-- radix__flush_tlb_range_psize(mm, start, end, psize);
-- else
-- radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
-+ radix__flush_tlb_pwc_range_psize(mm, start,
-+ end, psize, tlb->freed_tables);
- }
- }
-
-@@ -1254,16 +1249,10 @@ static void __radix__flush_tlb_range_psize(struct mm_struct *mm,
- preempt_enable();
- }
-
--void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
-- unsigned long end, int psize)
-+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
-+ unsigned long end, int psize, bool flush_pwc)
- {
-- return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
--}
--
--static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
-- unsigned long end, int psize)
--{
-- __radix__flush_tlb_range_psize(mm, start, end, psize, true);
-+ __radix__flush_tlb_range_psize(mm, start, end, psize, flush_pwc);
- }
-
- #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-@@ -1315,9 +1304,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
- #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
-
- void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
-- unsigned long start, unsigned long end)
-+ unsigned long start, unsigned long end,
-+ bool flush_pwc)
- {
-- radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
-+ __radix__flush_tlb_range_psize(vma->vm_mm, start,
-+ end, MMU_PAGE_2M, flush_pwc);
- }
- EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
-
+ pud_populate(mm, new_pud, (pmd_t *)pud_page_vaddr(pud));
+- flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE);
++ flush_pte_tlb_pwc_range(vma, old_addr, old_addr + PUD_SIZE, true);
+ if (new_ptl != old_ptl)
+ spin_unlock(new_ptl);
+ spin_unlock(old_ptl);
--
-2.31.1
+2.30.2