Excerpts from Aneesh Kumar K.V's message of March 15, 2021 9:38 pm: > Some architectures do have the concept of page walk cache and only mmu gather > interface supports flushing them. A fast mremap that involves moving page > table pages instead of copying pte entries should flush page walk cache since > the old translation cache is no more valid. Hence switch to mm gather to flush > TLB and mark tlb.freed_tables = 1. No page table pages need to be freed here. > With this the tlb flush is done outside page table lock (ptl). I would maybe just get archs that implement it to provide a specific flush_tlb+pwc_range for it, or else they get flush_tlb_range by default. I think that would be simpler for now, at least in generic code. There was some other talk of consolidating the TLB flush APIs, I jsut don't know if it's the best way to go to use the page/page table gathering and freeing API for it. Thanks, Nick > > Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxx> > --- > mm/mremap.c | 33 +++++++++++++++++++++++++++++---- > 1 file changed, 29 insertions(+), 4 deletions(-) > > diff --git a/mm/mremap.c b/mm/mremap.c > index 574287f9bb39..fafa73b965d3 100644 > --- a/mm/mremap.c > +++ b/mm/mremap.c > @@ -216,6 +216,7 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, > { > spinlock_t *old_ptl, *new_ptl; > struct mm_struct *mm = vma->vm_mm; > + struct mmu_gather tlb; > pmd_t pmd; > > /* > @@ -244,11 +245,12 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, > if (WARN_ON_ONCE(!pmd_none(*new_pmd))) > return false; > > + tlb_gather_mmu(&tlb, mm); > /* > * We don't have to worry about the ordering of src and dst > * ptlocks because exclusive mmap_lock prevents deadlock. > */ > - old_ptl = pmd_lock(vma->vm_mm, old_pmd); > + old_ptl = pmd_lock(mm, old_pmd); > new_ptl = pmd_lockptr(mm, new_pmd); > if (new_ptl != old_ptl) > spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); > @@ -257,13 +259,23 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, > pmd = *old_pmd; > pmd_clear(old_pmd); > > + /* > + * Mark the range. We are not freeing page table pages nor > + * regular pages. Hence we don't need to call tlb_remove_table() > + * or tlb_remove_page(). > + */ > + tlb_flush_pte_range(&tlb, old_addr, PMD_SIZE); > + tlb.freed_tables = 1; > VM_BUG_ON(!pmd_none(*new_pmd)); > pmd_populate(mm, new_pmd, (pgtable_t)pmd_page_vaddr(pmd)); > > - flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); > if (new_ptl != old_ptl) > spin_unlock(new_ptl); > spin_unlock(old_ptl); > + /* > + * This will invalidate both the old TLB and page table walk caches. > + */ > + tlb_finish_mmu(&tlb); > > return true; > } > @@ -282,6 +294,7 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, > { > spinlock_t *old_ptl, *new_ptl; > struct mm_struct *mm = vma->vm_mm; > + struct mmu_gather tlb; > pud_t pud; > > /* > @@ -291,11 +304,12 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, > if (WARN_ON_ONCE(!pud_none(*new_pud))) > return false; > > + tlb_gather_mmu(&tlb, mm); > /* > * We don't have to worry about the ordering of src and dst > * ptlocks because exclusive mmap_lock prevents deadlock. > */ > - old_ptl = pud_lock(vma->vm_mm, old_pud); > + old_ptl = pud_lock(mm, old_pud); > new_ptl = pud_lockptr(mm, new_pud); > if (new_ptl != old_ptl) > spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); > @@ -304,14 +318,25 @@ static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, > pud = *old_pud; > pud_clear(old_pud); > > + /* > + * Mark the range. We are not freeing page table pages nor > + * regular pages. Hence we don't need to call tlb_remove_table() > + * or tlb_remove_page(). > + */ > + tlb_flush_pte_range(&tlb, old_addr, PUD_SIZE); > + tlb.freed_tables = 1; > VM_BUG_ON(!pud_none(*new_pud)); > > pud_populate(mm, new_pud, (pmd_t *)pud_page_vaddr(pud)); > - flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE); > + > if (new_ptl != old_ptl) > spin_unlock(new_ptl); > spin_unlock(old_ptl); > > + /* > + * This will invalidate both the old TLB and page table walk caches. > + */ > + tlb_finish_mmu(&tlb); > return true; > } > #else > -- > 2.29.2 > >