[PATCH v4 5/9] powerpc/mm/book3s64: Update tlb flush routines to take a page walk cache flush argument
From: Aneesh Kumar K.V <hidden>
Date: 2021-04-14 09:02:04
Subsystem:
linux for powerpc (32-bit and 64-bit), the rest · Maintainers:
Madhavan Srinivasan, Michael Ellerman, Linus Torvalds
Signed-off-by: Aneesh Kumar K.V <redacted> --- .../include/asm/book3s/64/tlbflush-radix.h | 19 +++++++----- arch/powerpc/include/asm/book3s/64/tlbflush.h | 23 ++++++++++++--- arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 4 +-- arch/powerpc/mm/book3s64/radix_tlb.c | 29 +++++++------------ 4 files changed, 42 insertions(+), 33 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 8b33601cdb9d..171441a43b35 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h@@ -56,15 +56,18 @@ static inline void radix__flush_all_lpid_guest(unsigned int lpid) } #endif -extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end); -extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, - unsigned long end, int psize); -extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end); -extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, +void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end, + bool flush_pwc); +void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end, + bool flush_pwc); +void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, + unsigned long end, int psize, bool flush_pwc); +void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); +void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end); + extern void radix__local_flush_tlb_mm(struct mm_struct *mm); extern void radix__local_flush_all_mm(struct mm_struct *mm);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 215973b4cb26..f9f8a3a264f7 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h@@ -45,13 +45,30 @@ static inline void tlbiel_all_lpid(bool radix) hash__tlbiel_all(TLB_INVAL_SCOPE_LPID); } +static inline void flush_pmd_tlb_pwc_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end, + bool flush_pwc) +{ + if (radix_enabled()) + return radix__flush_pmd_tlb_range(vma, start, end, flush_pwc); + return hash__flush_tlb_range(vma, start, end); +} #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE static inline void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) +{ + return flush_pmd_tlb_pwc_range(vma, start, end, false); +} + +static inline void flush_hugetlb_tlb_pwc_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end, + bool flush_pwc) { if (radix_enabled()) - return radix__flush_pmd_tlb_range(vma, start, end); + return radix__flush_hugetlb_tlb_range(vma, start, end, flush_pwc); return hash__flush_tlb_range(vma, start, end); }
@@ -60,9 +77,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - if (radix_enabled()) - return radix__flush_hugetlb_tlb_range(vma, start, end); - return hash__flush_tlb_range(vma, start, end); + return flush_hugetlb_tlb_pwc_range(vma, start, end, false); } static inline void flush_tlb_range(struct vm_area_struct *vma,
diff --git a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
index cb91071eef52..e62f5679b119 100644
--- a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
+++ b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c@@ -26,13 +26,13 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v } void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) + unsigned long end, bool flush_pwc) { int psize; struct hstate *hstate = hstate_file(vma->vm_file); psize = hstate_get_psize(hstate); - radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize); + radix__flush_tlb_pwc_range_psize(vma->vm_mm, start, end, psize, flush_pwc); } /*
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 817a02ef6032..5a59e19f9e53 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c@@ -1090,7 +1090,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, { #ifdef CONFIG_HUGETLB_PAGE if (is_vm_hugetlb_page(vma)) - return radix__flush_hugetlb_tlb_range(vma, start, end); + return radix__flush_hugetlb_tlb_range(vma, start, end, false); #endif __radix__flush_tlb_range(vma->vm_mm, start, end);
@@ -1151,9 +1151,6 @@ void radix__flush_all_lpid_guest(unsigned int lpid) _tlbie_lpid_guest(lpid, RIC_FLUSH_ALL); } -static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, - unsigned long end, int psize); - void radix__tlb_flush(struct mmu_gather *tlb) { int psize = 0;
@@ -1177,10 +1174,8 @@ void radix__tlb_flush(struct mmu_gather *tlb) else radix__flush_all_mm(mm); } else { - if (!tlb->freed_tables) - radix__flush_tlb_range_psize(mm, start, end, psize); - else - radix__flush_tlb_pwc_range_psize(mm, start, end, psize); + radix__flush_tlb_pwc_range_psize(mm, start, + end, psize, tlb->freed_tables); } }
@@ -1254,16 +1249,10 @@ static void __radix__flush_tlb_range_psize(struct mm_struct *mm, preempt_enable(); } -void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, - unsigned long end, int psize) +void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, + unsigned long end, int psize, bool flush_pwc) { - return __radix__flush_tlb_range_psize(mm, start, end, psize, false); -} - -static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start, - unsigned long end, int psize) -{ - __radix__flush_tlb_range_psize(mm, start, end, psize, true); + __radix__flush_tlb_range_psize(mm, start, end, psize, flush_pwc); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -1315,9 +1304,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) + unsigned long start, unsigned long end, + bool flush_pwc) { - radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M); + __radix__flush_tlb_range_psize(vma->vm_mm, start, + end, MMU_PAGE_2M, flush_pwc); } EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
--
2.30.2