Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxx>
---
.../include/asm/book3s/64/tlbflush-radix.h | 19 ++++++++-------
arch/powerpc/include/asm/book3s/64/tlbflush.h | 23 +++++++++++++++----
arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 4 ++--
arch/powerpc/mm/book3s64/radix_tlb.c | 23 ++++++++-----------
4 files changed, 42 insertions(+), 27 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 8b33601cdb9d..90c91f7b526f 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -56,15 +56,18 @@ static inline void radix__flush_all_lpid_guest(unsigned int lpid)
}
#endif
-extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize);
-extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool also_pwc);
+void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool also_pwc);
+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize, bool also_pwc);
+void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
-extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
+void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
+
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_all_mm(struct mm_struct *mm);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 215973b4cb26..efe5336e2b6f 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -45,13 +45,30 @@ static inline void tlbiel_all_lpid(bool radix)
hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
}
+static inline void flush_pmd_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
+ bool also_pwc)
+{
+ if (radix_enabled())
+ return radix__flush_pmd_tlb_range(vma, start, end, also_pwc);
+ return hash__flush_tlb_range(vma, start, end);
+}
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
+{
+ return flush_pmd_tlb_pwc_range(vma, start, end, false);
+}
+
+static inline void flush_hugetlb_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
+ bool also_pwc)
{
if (radix_enabled())
- return radix__flush_pmd_tlb_range(vma, start, end);
+ return radix__flush_hugetlb_tlb_range(vma, start, end, also_pwc);
return hash__flush_tlb_range(vma, start, end);
}
@@ -60,9 +77,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
unsigned long start,
unsigned long end)
{
- if (radix_enabled())
- return radix__flush_hugetlb_tlb_range(vma, start, end);
- return hash__flush_tlb_range(vma, start, end);
+ return flush_hugetlb_tlb_pwc_range(vma, start, end, false);
}
static inline void flush_tlb_range(struct vm_area_struct *vma,
diff --git a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
index cb91071eef52..55c5c9c39ae2 100644
--- a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
+++ b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
@@ -26,13 +26,13 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v
}
void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
- unsigned long end)
+ unsigned long end, bool also_pwc)
{
int psize;
struct hstate *hstate = hstate_file(vma->vm_file);
psize = hstate_get_psize(hstate);
- radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize);
+ radix__flush_tlb_pwc_range_psize(vma->vm_mm, start, end, psize, also_pwc);
}
/*
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 817a02ef6032..416fe9b48e14 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -1090,7 +1090,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
{
#ifdef CONFIG_HUGETLB_PAGE
if (is_vm_hugetlb_page(vma))
- return radix__flush_hugetlb_tlb_range(vma, start, end);
+ return radix__flush_hugetlb_tlb_range(vma, start, end, false);
#endif
__radix__flush_tlb_range(vma->vm_mm, start, end);
@@ -1151,9 +1151,6 @@ void radix__flush_all_lpid_guest(unsigned int lpid)
_tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
}
-static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize);
-
void radix__tlb_flush(struct mmu_gather *tlb)
{
int psize = 0;
@@ -1177,10 +1174,8 @@ void radix__tlb_flush(struct mmu_gather *tlb)
else
radix__flush_all_mm(mm);
} else {
- if (!tlb->freed_tables)
- radix__flush_tlb_range_psize(mm, start, end, psize);
- else
- radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
+ radix__flush_tlb_pwc_range_psize(mm, start,
+ end, psize, tlb->freed_tables);
}
}
@@ -1260,10 +1255,10 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
}
-static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize)
+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize, bool also_pwc)
{
- __radix__flush_tlb_range_psize(mm, start, end, psize, true);
+ __radix__flush_tlb_range_psize(mm, start, end, psize, also_pwc);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -1315,9 +1310,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+ unsigned long start, unsigned long end,
+ bool also_pwc)
{
- radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
+ __radix__flush_tlb_range_psize(vma->vm_mm, start,
+ end, MMU_PAGE_2M, also_pwc);
}
EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
--
2.30.2
