When we use mmu_gather to batch flush tlb and free PTE pages, the TLB is
not flushed before pmd lock is unlocked. This may result in the following
two situations:
1) Userland can trigger page fault and fill a huge page, which will cause
the existence of small size TLB and huge TLB for the same address.
2) Userland can also trigger page fault and fill a PTE page, which will
cause the existence of two small size TLBs, but the PTE page they map
are different.
According to Intel's TLB Application note (317080), some CPUs of x86 do
not allow the 1) case, so define arch_flush_tlb_before_set_huge_page to
detect and fix this issue.
Signed-off-by: Qi Zheng <zhengqi.arch@xxxxxxxxxxxxx>
---
arch/x86/include/asm/pgtable.h | 6 ++++++
arch/x86/mm/pgtable.c | 13 +++++++++++++
2 files changed, 19 insertions(+)
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index e39311a89bf4..f93d964ab6a3 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -1668,6 +1668,12 @@ void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte);
#define arch_check_zapped_pmd arch_check_zapped_pmd
void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd);
+#ifdef CONFIG_PT_RECLAIM
+#define arch_flush_tlb_before_set_huge_page arch_flush_tlb_before_set_huge_page
+void arch_flush_tlb_before_set_huge_page(struct mm_struct *mm,
+ unsigned long addr);
+#endif
+
#ifdef CONFIG_XEN_PV
#define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young
static inline bool arch_has_hw_nonleaf_pmd_young(void)
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index cd5bf2157611..d037f7425f82 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -926,3 +926,16 @@ void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd)
VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
pmd_shstk(pmd));
}
+
+#ifdef CONFIG_PT_RECLAIM
+void arch_flush_tlb_before_set_huge_page(struct mm_struct *mm,
+ unsigned long addr)
+{
+ if (atomic_read(&mm->tlb_flush_pending)) {
+ unsigned long start = ALIGN_DOWN(addr, PMD_SIZE);
+ unsigned long end = start + PMD_SIZE;
+
+ flush_tlb_mm_range(mm, start, end, PAGE_SHIFT, false);
+ }
+}
+#endif /* CONFIG_PT_RECLAIM */
--
2.20.1
