tlb_remove_huge_tlb_entry only considers PMD_SIZE and PUD_SIZE when updating the mmu_gather structure. Unfortunately on arm64 there are two additional huge page sizes that need to be covered: CONT_PTE_SIZE and CONT_PMD_SIZE. Where an end-user attempts to employ contiguous huge pages, a VM_BUG_ON can be experienced due to the fact that the tlb structure hasn't been correctly updated by the relevant tlb_flush_p.._range() call from tlb_remove_huge_tlb_entry. This patch adds the ability to define per-arch implementations of tlb_remove_huge_tlb_entry and supplies an arm64 implementation that covers the contiguous hint sizes. Reported-by: David Hildenbrand <david@xxxxxxxxxx> Cc: Anshuman Khandual <anshuman.khandual@xxxxxxx> Cc: Catalin Marinas <catalin.marinas@xxxxxxx> Cc: Will Deacon <will@xxxxxxxxxx> Cc: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx> Link: https://lore.kernel.org/linux-mm/811c5c8e-b3a2-85d2-049c-717f17c3a03a@xxxxxxxxxx/ Signed-off-by: Steve Capper <steve.capper@xxxxxxx> --- arch/arm64/include/asm/tlb.h | 14 ++++++++++++++ include/asm-generic/tlb.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index c995d1f4594f..311d201d4e72 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -19,6 +19,20 @@ static inline void __tlb_remove_table(void *_table) #define tlb_flush tlb_flush static void tlb_flush(struct mmu_gather *tlb); +#define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ + do { \ + unsigned long _sz = huge_page_size(h); \ + if (_sz == CONT_PTE_SIZE) \ + tlb_flush_pte_range(tlb, address, _sz); \ + else if (_sz == PMD_SIZE) \ + tlb_flush_pmd_range(tlb, address, _sz); \ + else if (_sz == CONT_PMD_SIZE) \ + tlb_flush_pmd_range(tlb, address, _sz); \ + else if (_sz == PUD_SIZE) \ + tlb_flush_pud_range(tlb, address, _sz); \ + __tlb_remove_tlb_entry(tlb, ptep, address); \ + } while (0) + #include <asm-generic/tlb.h> /* diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 2c68a545ffa7..4622ee45f739 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -562,6 +562,7 @@ static inline void tlb_flush_p4d_range(struct mmu_gather *tlb, __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) +#ifndef tlb_remove_huge_tlb_entry #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address) \ do { \ unsigned long _sz = huge_page_size(h); \ @@ -571,6 +572,7 @@ static inline void tlb_flush_p4d_range(struct mmu_gather *tlb, tlb_flush_pud_range(tlb, address, _sz); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) +#endif /** * tlb_remove_pmd_tlb_entry - remember a pmd mapping for later tlb invalidation -- 2.35.1
