This allows arch which need to do special handing with respect to different page size when flushing tlb to implement the same in mmu gather Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxxxxxxx> --- include/asm-generic/tlb.h | 11 +++++++--- mm/huge_memory.c | 2 +- mm/hugetlb.c | 54 ++++++++++++++++++----------------------------- mm/memory.c | 38 +++++++++++++++++++++++++++------ 4 files changed, 61 insertions(+), 44 deletions(-) diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index 9dbb739cafa0..876ceb44a3bb 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -107,6 +107,7 @@ struct mmu_gather { struct mmu_gather_batch local; struct page *__pages[MMU_GATHER_BUNDLE]; unsigned int batch_count; + int page_size; }; #define HAVE_GENERIC_MMU_GATHER @@ -115,16 +116,20 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long void tlb_flush_mmu(struct mmu_gather *tlb); void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end); -int __tlb_remove_page(struct mmu_gather *tlb, struct page *page); +bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page, int page_size); /* tlb_remove_page * Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when * required. */ -static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) +static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page, + int page_size) { - if (!__tlb_remove_page(tlb, page)) + if (__tlb_remove_page(tlb, page, page_size)) { tlb_flush_mmu(tlb); + tlb->page_size = page_size; + __tlb_remove_page(tlb, page, page_size); + } } static inline void __tlb_adjust_range(struct mmu_gather *tlb, diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 36c22a89df61..c1c067e94479 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1669,7 +1669,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); atomic_long_dec(&tlb->mm->nr_ptes); spin_unlock(ptl); - tlb_remove_page(tlb, page); + tlb_remove_page(tlb, page, HPAGE_PMD_SIZE); } return 1; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d1b07387c5db..d158ceb89ead 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3121,7 +3121,6 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, unsigned long start, unsigned long end, struct page *ref_page) { - int force_flush = 0; struct mm_struct *mm = vma->vm_mm; unsigned long address; pte_t *ptep; @@ -3140,19 +3139,22 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb_start_vma(tlb, vma); mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); address = start; -again: for (; address < end; address += sz) { ptep = huge_pte_offset(mm, address); if (!ptep) continue; ptl = huge_pte_lock(h, mm, ptep); - if (huge_pmd_unshare(mm, &address, ptep)) - goto unlock; + if (huge_pmd_unshare(mm, &address, ptep)) { + spin_unlock(ptl); + continue; + } pte = huge_ptep_get(ptep); - if (huge_pte_none(pte)) - goto unlock; + if (huge_pte_none(pte)) { + spin_unlock(ptl); + continue; + } /* * Migrating hugepage or HWPoisoned hugepage is already @@ -3160,7 +3162,8 @@ again: */ if (unlikely(!pte_present(pte))) { huge_pte_clear(mm, address, ptep); - goto unlock; + spin_unlock(ptl); + continue; } page = pte_page(pte); @@ -3170,9 +3173,10 @@ again: * are about to unmap is the actual page of interest. */ if (ref_page) { - if (page != ref_page) - goto unlock; - + if (page != ref_page) { + spin_unlock(ptl); + continue; + } /* * Mark the VMA as having unmapped its page so that * future faults in this VMA will fail rather than @@ -3188,30 +3192,14 @@ again: hugetlb_count_sub(pages_per_huge_page(h), mm); page_remove_rmap(page, true); - force_flush = !__tlb_remove_page(tlb, page); - if (force_flush) { - address += sz; - spin_unlock(ptl); - break; - } - /* Bail out after unmapping reference page if supplied */ - if (ref_page) { - spin_unlock(ptl); - break; - } -unlock: + spin_unlock(ptl); - } - /* - * mmu_gather ran out of room to batch pages, we break out of - * the PTE lock to avoid doing the potential expensive TLB invalidate - * and page-free while holding it. - */ - if (force_flush) { - force_flush = 0; - tlb_flush_mmu(tlb); - if (address < end && !ref_page) - goto again; + tlb_remove_page(tlb, page, huge_page_size(h)); + /* + * Bail out after unmapping reference page if supplied + */ + if (ref_page) + break; } mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); tlb_end_vma(tlb, vma); diff --git a/mm/memory.c b/mm/memory.c index 635451abc8f7..7700edf9e243 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -231,6 +231,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long #ifdef CONFIG_HAVE_RCU_TABLE_FREE tlb->batch = NULL; #endif + tlb->page_size = 0 ; __tlb_reset_range(tlb); } @@ -290,23 +291,39 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e * handling the additional races in SMP caused by other CPUs caching valid * mappings in their TLBs. Returns the number of free page slots left. * When out of page slots we must call tlb_flush_mmu(). + *returns true if the caller should flush. */ -int __tlb_remove_page(struct mmu_gather *tlb, struct page *page) +bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page, int page_size) { struct mmu_gather_batch *batch; VM_BUG_ON(!tlb->end); + if (!tlb->page_size) + tlb->page_size = page_size; + else { + if (page_size != tlb->page_size) + return true; + } + batch = tlb->active; - batch->pages[batch->nr++] = page; if (batch->nr == batch->max) { if (!tlb_next_batch(tlb)) - return 0; + return true; batch = tlb->active; } VM_BUG_ON_PAGE(batch->nr > batch->max, page); - return batch->max - batch->nr; + batch->pages[batch->nr++] = page; + return false; +} + +int __tlb_remove_pte_page(struct mmu_gather *tlb, struct page *page) +{ + /* active->nr should be zero when we call this */ + VM_BUG_ON_PAGE(tlb->active->nr, page); + tlb->page_size = PAGE_SIZE; + return __tlb_remove_page(tlb, page, PAGE_SIZE); } #endif /* HAVE_GENERIC_MMU_GATHER */ @@ -1071,6 +1088,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, pte_t *start_pte; pte_t *pte; swp_entry_t entry; + struct page *delayed_page = NULL; again: init_rss_vec(rss); @@ -1116,8 +1134,9 @@ again: page_remove_rmap(page, false); if (unlikely(page_mapcount(page) < 0)) print_bad_pte(vma, addr, ptent, page); - if (unlikely(!__tlb_remove_page(tlb, page))) { + if (unlikely(__tlb_remove_page(tlb, page, PAGE_SIZE))) { force_flush = 1; + delayed_page = page; addr += PAGE_SIZE; break; } @@ -1158,9 +1177,14 @@ again: if (force_flush) { force_flush = 0; tlb_flush_mmu_free(tlb); - - if (addr != end) + if (delayed_page) { + /* remove the page with new size */ + __tlb_remove_pte_page(tlb, delayed_page); + delayed_page = NULL; + } + if (addr != end) { goto again; + } } return addr; -- 2.5.0 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@xxxxxxxxx. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@xxxxxxxxx"> email@xxxxxxxxx </a>