On Wed 05-12-18 00:36:27, jglisse@xxxxxxxxxx wrote:
> From: Jérôme Glisse <jglisse@xxxxxxxxxx>
>
> To avoid having to change many call sites everytime we want to add a
> parameter use a structure to group all parameters for the mmu_notifier
> invalidate_range_start/end cakks. No functional changes with this
> patch.
>
> Changes since v1:
> - introduce mmu_notifier_range_init() as an helper to initialize
> the range structure allowing to optimize out the case when mmu
> notifier is not enabled
> - fix mm/migrate.c migrate_vma_collect()
>
> Signed-off-by: Jérôme Glisse <jglisse@xxxxxxxxxx>
> Acked-by: Christian König <christian.koenig@xxxxxxx>
The patch looks good to me. You can add:
Acked-by: Jan Kara <jack@xxxxxxx>
Honza
> Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
> Cc: Matthew Wilcox <mawilcox@xxxxxxxxxxxxx>
> Cc: Ross Zwisler <zwisler@xxxxxxxxxx>
> Cc: Jan Kara <jack@xxxxxxx>
> Cc: Dan Williams <dan.j.williams@xxxxxxxxx>
> Cc: Paolo Bonzini <pbonzini@xxxxxxxxxx>
> Cc: Radim Krčmář <rkrcmar@xxxxxxxxxx>
> Cc: Michal Hocko <mhocko@xxxxxxxxxx>
> Cc: Felix Kuehling <felix.kuehling@xxxxxxx>
> Cc: Ralph Campbell <rcampbell@xxxxxxxxxx>
> Cc: John Hubbard <jhubbard@xxxxxxxxxx>
> Cc: kvm@xxxxxxxxxxxxxxx
> Cc: dri-devel@xxxxxxxxxxxxxxxxxxxxx
> Cc: linux-rdma@xxxxxxxxxxxxxxx
> Cc: linux-fsdevel@xxxxxxxxxxxxxxx
> ---
> fs/dax.c | 8 +--
> fs/proc/task_mmu.c | 7 ++-
> include/linux/mm.h | 4 +-
> include/linux/mmu_notifier.h | 87 +++++++++++++++++++++-----------
> kernel/events/uprobes.c | 10 ++--
> mm/huge_memory.c | 54 ++++++++++----------
> mm/hugetlb.c | 52 ++++++++++---------
> mm/khugepaged.c | 10 ++--
> mm/ksm.c | 21 ++++----
> mm/madvise.c | 21 ++++----
> mm/memory.c | 97 ++++++++++++++++++------------------
> mm/migrate.c | 25 +++++-----
> mm/mmu_notifier.c | 35 +++----------
> mm/mprotect.c | 15 +++---
> mm/mremap.c | 10 ++--
> mm/oom_kill.c | 17 ++++---
> mm/rmap.c | 30 ++++++-----
> 17 files changed, 258 insertions(+), 245 deletions(-)
>
> diff --git a/fs/dax.c b/fs/dax.c
> index 9bcce89ea18e..874085bacaf5 100644
> --- a/fs/dax.c
> +++ b/fs/dax.c
> @@ -758,7 +758,8 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
>
> i_mmap_lock_read(mapping);
> vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
> - unsigned long address, start, end;
> + struct mmu_notifier_range range;
> + unsigned long address;
>
> cond_resched();
>
> @@ -772,7 +773,8 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
> * call mmu_notifier_invalidate_range_start() on our behalf
> * before taking any lock.
> */
> - if (follow_pte_pmd(vma->vm_mm, address, &start, &end, &ptep, &pmdp, &ptl))
> + if (follow_pte_pmd(vma->vm_mm, address, &range,
> + &ptep, &pmdp, &ptl))
> continue;
>
> /*
> @@ -814,7 +816,7 @@ static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
> pte_unmap_unlock(ptep, ptl);
> }
>
> - mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
> + mmu_notifier_invalidate_range_end(&range);
> }
> i_mmap_unlock_read(mapping);
> }
> diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
> index 47c3764c469b..b3ddceb003bc 100644
> --- a/fs/proc/task_mmu.c
> +++ b/fs/proc/task_mmu.c
> @@ -1096,6 +1096,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
> return -ESRCH;
> mm = get_task_mm(task);
> if (mm) {
> + struct mmu_notifier_range range;
> struct clear_refs_private cp = {
> .type = type,
> };
> @@ -1139,11 +1140,13 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
> downgrade_write(&mm->mmap_sem);
> break;
> }
> - mmu_notifier_invalidate_range_start(mm, 0, -1);
> +
> + mmu_notifier_range_init(&range, mm, 0, -1UL);
> + mmu_notifier_invalidate_range_start(&range);
> }
> walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
> if (type == CLEAR_REFS_SOFT_DIRTY)
> - mmu_notifier_invalidate_range_end(mm, 0, -1);
> + mmu_notifier_invalidate_range_end(&range);
> tlb_finish_mmu(&tlb, 0, -1);
> up_read(&mm->mmap_sem);
> out_mm:
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index 5411de93a363..e7b6f2b30713 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1397,6 +1397,8 @@ struct mm_walk {
> void *private;
> };
>
> +struct mmu_notifier_range;
> +
> int walk_page_range(unsigned long addr, unsigned long end,
> struct mm_walk *walk);
> int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
> @@ -1405,7 +1407,7 @@ void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
> int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
> struct vm_area_struct *vma);
> int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
> - unsigned long *start, unsigned long *end,
> + struct mmu_notifier_range *range,
> pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp);
> int follow_pfn(struct vm_area_struct *vma, unsigned long address,
> unsigned long *pfn);
> diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
> index 368f0c1a049d..39b06772427f 100644
> --- a/include/linux/mmu_notifier.h
> +++ b/include/linux/mmu_notifier.h
> @@ -220,11 +220,8 @@ extern int __mmu_notifier_test_young(struct mm_struct *mm,
> unsigned long address);
> extern void __mmu_notifier_change_pte(struct mm_struct *mm,
> unsigned long address, pte_t pte);
> -extern int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end,
> - bool blockable);
> -extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end,
> +extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *);
> +extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r,
> bool only_end);
> extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
> unsigned long start, unsigned long end);
> @@ -268,33 +265,37 @@ static inline void mmu_notifier_change_pte(struct mm_struct *mm,
> __mmu_notifier_change_pte(mm, address, pte);
> }
>
> -static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline void
> +mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
> {
> - if (mm_has_notifiers(mm))
> - __mmu_notifier_invalidate_range_start(mm, start, end, true);
> + if (mm_has_notifiers(range->mm)) {
> + range->blockable = true;
> + __mmu_notifier_invalidate_range_start(range);
> + }
> }
>
> -static inline int mmu_notifier_invalidate_range_start_nonblock(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline int
> +mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
> {
> - if (mm_has_notifiers(mm))
> - return __mmu_notifier_invalidate_range_start(mm, start, end, false);
> + if (mm_has_notifiers(range->mm)) {
> + range->blockable = false;
> + return __mmu_notifier_invalidate_range_start(range);
> + }
> return 0;
> }
>
> -static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline void
> +mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
> {
> - if (mm_has_notifiers(mm))
> - __mmu_notifier_invalidate_range_end(mm, start, end, false);
> + if (mm_has_notifiers(range->mm))
> + __mmu_notifier_invalidate_range_end(range, false);
> }
>
> -static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline void
> +mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
> {
> - if (mm_has_notifiers(mm))
> - __mmu_notifier_invalidate_range_end(mm, start, end, true);
> + if (mm_has_notifiers(range->mm))
> + __mmu_notifier_invalidate_range_end(range, true);
> }
>
> static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
> @@ -315,6 +316,17 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
> __mmu_notifier_mm_destroy(mm);
> }
>
> +
> +static inline void mmu_notifier_range_init(struct mmu_notifier_range *range,
> + struct mm_struct *mm,
> + unsigned long start,
> + unsigned long end)
> +{
> + range->mm = mm;
> + range->start = start;
> + range->end = end;
> +}
> +
> #define ptep_clear_flush_young_notify(__vma, __address, __ptep) \
> ({ \
> int __young; \
> @@ -428,6 +440,23 @@ extern void mmu_notifier_synchronize(void);
>
> #else /* CONFIG_MMU_NOTIFIER */
>
> +struct mmu_notifier_range {
> + unsigned long start;
> + unsigned long end;
> +};
> +
> +static inline void _mmu_notifier_range_init(struct mmu_notifier_range *range,
> + unsigned long start,
> + unsigned long end)
> +{
> + range->start = start;
> + range->end = end;
> +}
> +
> +#define mmu_notifier_range_init(range, mm, start, end) \
> + _mmu_notifier_range_init(range, start, end)
> +
> +
> static inline int mm_has_notifiers(struct mm_struct *mm)
> {
> return 0;
> @@ -455,24 +484,24 @@ static inline void mmu_notifier_change_pte(struct mm_struct *mm,
> {
> }
>
> -static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline void
> +mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
> {
> }
>
> -static inline int mmu_notifier_invalidate_range_start_nonblock(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline int
> +mmu_notifier_invalidate_range_start_nonblock(struct mmu_notifier_range *range)
> {
> return 0;
> }
>
> -static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline
> +void mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range)
> {
> }
>
> -static inline void mmu_notifier_invalidate_range_only_end(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> +static inline void
> +mmu_notifier_invalidate_range_only_end(struct mmu_notifier_range *range)
> {
> }
>
> diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
> index 322e97bbb437..1fc8a93709c3 100644
> --- a/kernel/events/uprobes.c
> +++ b/kernel/events/uprobes.c
> @@ -171,11 +171,11 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
> .address = addr,
> };
> int err;
> - /* For mmu_notifiers */
> - const unsigned long mmun_start = addr;
> - const unsigned long mmun_end = addr + PAGE_SIZE;
> + struct mmu_notifier_range range;
> struct mem_cgroup *memcg;
>
> + mmu_notifier_range_init(&range, mm, addr, addr + PAGE_SIZE);
> +
> VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page);
>
> err = mem_cgroup_try_charge(new_page, vma->vm_mm, GFP_KERNEL, &memcg,
> @@ -186,7 +186,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
> /* For try_to_free_swap() and munlock_vma_page() below */
> lock_page(old_page);
>
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_start(&range);
> err = -EAGAIN;
> if (!page_vma_mapped_walk(&pvmw)) {
> mem_cgroup_cancel_charge(new_page, memcg, false);
> @@ -220,7 +220,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
>
> err = 0;
> unlock:
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> unlock_page(old_page);
> return err;
> }
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index 622cced74fd9..c1d3ce809416 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -1144,8 +1144,7 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
> int i;
> vm_fault_t ret = 0;
> struct page **pages;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
>
> pages = kmalloc_array(HPAGE_PMD_NR, sizeof(struct page *),
> GFP_KERNEL);
> @@ -1183,9 +1182,9 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
> cond_resched();
> }
>
> - mmun_start = haddr;
> - mmun_end = haddr + HPAGE_PMD_SIZE;
> - mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, vma->vm_mm, haddr,
> + haddr + HPAGE_PMD_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
>
> vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
> if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
> @@ -1230,8 +1229,7 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
> * No need to double call mmu_notifier->invalidate_range() callback as
> * the above pmdp_huge_clear_flush_notify() did already call it.
> */
> - mmu_notifier_invalidate_range_only_end(vma->vm_mm, mmun_start,
> - mmun_end);
> + mmu_notifier_invalidate_range_only_end(&range);
>
> ret |= VM_FAULT_WRITE;
> put_page(page);
> @@ -1241,7 +1239,7 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
>
> out_free_pages:
> spin_unlock(vmf->ptl);
> - mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> for (i = 0; i < HPAGE_PMD_NR; i++) {
> memcg = (void *)page_private(pages[i]);
> set_page_private(pages[i], 0);
> @@ -1258,8 +1256,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
> struct page *page = NULL, *new_page;
> struct mem_cgroup *memcg;
> unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
> gfp_t huge_gfp; /* for allocation and charge */
> vm_fault_t ret = 0;
>
> @@ -1349,9 +1346,9 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
> vma, HPAGE_PMD_NR);
> __SetPageUptodate(new_page);
>
> - mmun_start = haddr;
> - mmun_end = haddr + HPAGE_PMD_SIZE;
> - mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, vma->vm_mm, haddr,
> + haddr + HPAGE_PMD_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
>
> spin_lock(vmf->ptl);
> if (page)
> @@ -1386,8 +1383,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
> * No need to double call mmu_notifier->invalidate_range() callback as
> * the above pmdp_huge_clear_flush_notify() did already call it.
> */
> - mmu_notifier_invalidate_range_only_end(vma->vm_mm, mmun_start,
> - mmun_end);
> + mmu_notifier_invalidate_range_only_end(&range);
> out:
> return ret;
> out_unlock:
> @@ -2028,14 +2024,15 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
> unsigned long address)
> {
> spinlock_t *ptl;
> - struct mm_struct *mm = vma->vm_mm;
> - unsigned long haddr = address & HPAGE_PUD_MASK;
> + struct mmu_notifier_range range;
>
> - mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PUD_SIZE);
> - ptl = pud_lock(mm, pud);
> + mmu_notifier_range_init(&range, vma->vm_mm, address & HPAGE_PUD_MASK,
> + (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
> + ptl = pud_lock(vma->vm_mm, pud);
> if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
> goto out;
> - __split_huge_pud_locked(vma, pud, haddr);
> + __split_huge_pud_locked(vma, pud, range.start);
>
> out:
> spin_unlock(ptl);
> @@ -2043,8 +2040,7 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
> * No need to double call mmu_notifier->invalidate_range() callback as
> * the above pudp_huge_clear_flush_notify() did already call it.
> */
> - mmu_notifier_invalidate_range_only_end(mm, haddr, haddr +
> - HPAGE_PUD_SIZE);
> + mmu_notifier_invalidate_range_only_end(&range);
> }
> #endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
>
> @@ -2244,11 +2240,12 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
> unsigned long address, bool freeze, struct page *page)
> {
> spinlock_t *ptl;
> - struct mm_struct *mm = vma->vm_mm;
> - unsigned long haddr = address & HPAGE_PMD_MASK;
> + struct mmu_notifier_range range;
>
> - mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
> - ptl = pmd_lock(mm, pmd);
> + mmu_notifier_range_init(&range, vma->vm_mm, address & HPAGE_PMD_MASK,
> + (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
> + ptl = pmd_lock(vma->vm_mm, pmd);
>
> /*
> * If caller asks to setup a migration entries, we need a page to check
> @@ -2264,7 +2261,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
> clear_page_mlock(page);
> } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))
> goto out;
> - __split_huge_pmd_locked(vma, pmd, haddr, freeze);
> + __split_huge_pmd_locked(vma, pmd, range.start, freeze);
> out:
> spin_unlock(ptl);
> /*
> @@ -2280,8 +2277,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
> * any further changes to individual pte will notify. So no need
> * to call mmu_notifier->invalidate_range()
> */
> - mmu_notifier_invalidate_range_only_end(mm, haddr, haddr +
> - HPAGE_PMD_SIZE);
> + mmu_notifier_invalidate_range_only_end(&range);
> }
>
> void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
> index 705a3e9cc910..e7c179cbcd75 100644
> --- a/mm/hugetlb.c
> +++ b/mm/hugetlb.c
> @@ -3239,16 +3239,16 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
> int cow;
> struct hstate *h = hstate_vma(vma);
> unsigned long sz = huge_page_size(h);
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
> int ret = 0;
>
> cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
>
> - mmun_start = vma->vm_start;
> - mmun_end = vma->vm_end;
> - if (cow)
> - mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
> + if (cow) {
> + mmu_notifier_range_init(&range, src, vma->vm_start,
> + vma->vm_end);
> + mmu_notifier_invalidate_range_start(&range);
> + }
>
> for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
> spinlock_t *src_ptl, *dst_ptl;
> @@ -3324,7 +3324,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
> }
>
> if (cow)
> - mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
>
> return ret;
> }
> @@ -3341,8 +3341,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
> struct page *page;
> struct hstate *h = hstate_vma(vma);
> unsigned long sz = huge_page_size(h);
> - unsigned long mmun_start = start; /* For mmu_notifiers */
> - unsigned long mmun_end = end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
>
> WARN_ON(!is_vm_hugetlb_page(vma));
> BUG_ON(start & ~huge_page_mask(h));
> @@ -3358,8 +3357,9 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
> /*
> * If sharing possible, alert mmu notifiers of worst case.
> */
> - adjust_range_if_pmd_sharing_possible(vma, &mmun_start, &mmun_end);
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, mm, start, end);
> + adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
> + mmu_notifier_invalidate_range_start(&range);
> address = start;
> for (; address < end; address += sz) {
> ptep = huge_pte_offset(mm, address, sz);
> @@ -3427,7 +3427,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
> if (ref_page)
> break;
> }
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> tlb_end_vma(tlb, vma);
> }
>
> @@ -3545,9 +3545,8 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
> struct page *old_page, *new_page;
> int outside_reserve = 0;
> vm_fault_t ret = 0;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> unsigned long haddr = address & huge_page_mask(h);
> + struct mmu_notifier_range range;
>
> pte = huge_ptep_get(ptep);
> old_page = pte_page(pte);
> @@ -3626,9 +3625,8 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
> __SetPageUptodate(new_page);
> set_page_huge_active(new_page);
>
> - mmun_start = haddr;
> - mmun_end = mmun_start + huge_page_size(h);
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, mm, haddr, haddr + huge_page_size(h));
> + mmu_notifier_invalidate_range_start(&range);
>
> /*
> * Retake the page table lock to check for racing updates
> @@ -3641,7 +3639,7 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
>
> /* Break COW */
> huge_ptep_clear_flush(vma, haddr, ptep);
> - mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range(mm, range.start, range.end);
> set_huge_pte_at(mm, haddr, ptep,
> make_huge_pte(vma, new_page, 1));
> page_remove_rmap(old_page, true);
> @@ -3650,7 +3648,7 @@ static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
> new_page = old_page;
> }
> spin_unlock(ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> out_release_all:
> restore_reserve_on_error(h, vma, haddr, new_page);
> put_page(new_page);
> @@ -4339,21 +4337,21 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
> pte_t pte;
> struct hstate *h = hstate_vma(vma);
> unsigned long pages = 0;
> - unsigned long f_start = start;
> - unsigned long f_end = end;
> bool shared_pmd = false;
> + struct mmu_notifier_range range;
>
> /*
> * In the case of shared PMDs, the area to flush could be beyond
> - * start/end. Set f_start/f_end to cover the maximum possible
> + * start/end. Set range.start/range.end to cover the maximum possible
> * range if PMD sharing is possible.
> */
> - adjust_range_if_pmd_sharing_possible(vma, &f_start, &f_end);
> + mmu_notifier_range_init(&range, mm, start, end);
> + adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
>
> BUG_ON(address >= end);
> - flush_cache_range(vma, f_start, f_end);
> + flush_cache_range(vma, range.start, range.end);
>
> - mmu_notifier_invalidate_range_start(mm, f_start, f_end);
> + mmu_notifier_invalidate_range_start(&range);
> i_mmap_lock_write(vma->vm_file->f_mapping);
> for (; address < end; address += huge_page_size(h)) {
> spinlock_t *ptl;
> @@ -4404,7 +4402,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
> * did unshare a page of pmds, flush the range corresponding to the pud.
> */
> if (shared_pmd)
> - flush_hugetlb_tlb_range(vma, f_start, f_end);
> + flush_hugetlb_tlb_range(vma, range.start, range.end);
> else
> flush_hugetlb_tlb_range(vma, start, end);
> /*
> @@ -4414,7 +4412,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
> * See Documentation/vm/mmu_notifier.rst
> */
> i_mmap_unlock_write(vma->vm_file->f_mapping);
> - mmu_notifier_invalidate_range_end(mm, f_start, f_end);
> + mmu_notifier_invalidate_range_end(&range);
>
> return pages << h->order;
> }
> diff --git a/mm/khugepaged.c b/mm/khugepaged.c
> index 8e2ff195ecb3..7736f6c37f19 100644
> --- a/mm/khugepaged.c
> +++ b/mm/khugepaged.c
> @@ -944,8 +944,7 @@ static void collapse_huge_page(struct mm_struct *mm,
> int isolated = 0, result = 0;
> struct mem_cgroup *memcg;
> struct vm_area_struct *vma;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
> gfp_t gfp;
>
> VM_BUG_ON(address & ~HPAGE_PMD_MASK);
> @@ -1017,9 +1016,8 @@ static void collapse_huge_page(struct mm_struct *mm,
> pte = pte_offset_map(pmd, address);
> pte_ptl = pte_lockptr(mm, pmd);
>
> - mmun_start = address;
> - mmun_end = address + HPAGE_PMD_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, mm, address, address + HPAGE_PMD_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
> pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
> /*
> * After this gup_fast can't run anymore. This also removes
> @@ -1029,7 +1027,7 @@ static void collapse_huge_page(struct mm_struct *mm,
> */
> _pmd = pmdp_collapse_flush(vma, address, pmd);
> spin_unlock(pmd_ptl);
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
>
> spin_lock(pte_ptl);
> isolated = __collapse_huge_page_isolate(vma, address, pte);
> diff --git a/mm/ksm.c b/mm/ksm.c
> index 5b0894b45ee5..6239d2df7a8e 100644
> --- a/mm/ksm.c
> +++ b/mm/ksm.c
> @@ -1042,8 +1042,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
> };
> int swapped;
> int err = -EFAULT;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
>
> pvmw.address = page_address_in_vma(page, vma);
> if (pvmw.address == -EFAULT)
> @@ -1051,9 +1050,9 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
>
> BUG_ON(PageTransCompound(page));
>
> - mmun_start = pvmw.address;
> - mmun_end = pvmw.address + PAGE_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, mm, pvmw.address,
> + pvmw.address + PAGE_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
>
> if (!page_vma_mapped_walk(&pvmw))
> goto out_mn;
> @@ -1105,7 +1104,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
> out_unlock:
> page_vma_mapped_walk_done(&pvmw);
> out_mn:
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> out:
> return err;
> }
> @@ -1129,8 +1128,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
> spinlock_t *ptl;
> unsigned long addr;
> int err = -EFAULT;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
>
> addr = page_address_in_vma(page, vma);
> if (addr == -EFAULT)
> @@ -1140,9 +1138,8 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
> if (!pmd)
> goto out;
>
> - mmun_start = addr;
> - mmun_end = addr + PAGE_SIZE;
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, mm, addr, addr + PAGE_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
>
> ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
> if (!pte_same(*ptep, orig_pte)) {
> @@ -1188,7 +1185,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
> pte_unmap_unlock(ptep, ptl);
> err = 0;
> out_mn:
> - mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> out:
> return err;
> }
> diff --git a/mm/madvise.c b/mm/madvise.c
> index 6cb1ca93e290..21a7881a2db4 100644
> --- a/mm/madvise.c
> +++ b/mm/madvise.c
> @@ -458,29 +458,30 @@ static void madvise_free_page_range(struct mmu_gather *tlb,
> static int madvise_free_single_vma(struct vm_area_struct *vma,
> unsigned long start_addr, unsigned long end_addr)
> {
> - unsigned long start, end;
> struct mm_struct *mm = vma->vm_mm;
> + struct mmu_notifier_range range;
> struct mmu_gather tlb;
>
> /* MADV_FREE works for only anon vma at the moment */
> if (!vma_is_anonymous(vma))
> return -EINVAL;
>
> - start = max(vma->vm_start, start_addr);
> - if (start >= vma->vm_end)
> + range.start = max(vma->vm_start, start_addr);
> + if (range.start >= vma->vm_end)
> return -EINVAL;
> - end = min(vma->vm_end, end_addr);
> - if (end <= vma->vm_start)
> + range.end = min(vma->vm_end, end_addr);
> + if (range.end <= vma->vm_start)
> return -EINVAL;
> + mmu_notifier_range_init(&range, mm, range.start, range.end);
>
> lru_add_drain();
> - tlb_gather_mmu(&tlb, mm, start, end);
> + tlb_gather_mmu(&tlb, mm, range.start, range.end);
> update_hiwater_rss(mm);
>
> - mmu_notifier_invalidate_range_start(mm, start, end);
> - madvise_free_page_range(&tlb, vma, start, end);
> - mmu_notifier_invalidate_range_end(mm, start, end);
> - tlb_finish_mmu(&tlb, start, end);
> + mmu_notifier_invalidate_range_start(&range);
> + madvise_free_page_range(&tlb, vma, range.start, range.end);
> + mmu_notifier_invalidate_range_end(&range);
> + tlb_finish_mmu(&tlb, range.start, range.end);
>
> return 0;
> }
> diff --git a/mm/memory.c b/mm/memory.c
> index 4ad2d293ddc2..574307f11464 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -973,8 +973,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> unsigned long next;
> unsigned long addr = vma->vm_start;
> unsigned long end = vma->vm_end;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
> + struct mmu_notifier_range range;
> bool is_cow;
> int ret;
>
> @@ -1008,11 +1007,11 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> * is_cow_mapping() returns true.
> */
> is_cow = is_cow_mapping(vma->vm_flags);
> - mmun_start = addr;
> - mmun_end = end;
> - if (is_cow)
> - mmu_notifier_invalidate_range_start(src_mm, mmun_start,
> - mmun_end);
> +
> + if (is_cow) {
> + mmu_notifier_range_init(&range, src_mm, addr, end);
> + mmu_notifier_invalidate_range_start(&range);
> + }
>
> ret = 0;
> dst_pgd = pgd_offset(dst_mm, addr);
> @@ -1029,7 +1028,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
> } while (dst_pgd++, src_pgd++, addr = next, addr != end);
>
> if (is_cow)
> - mmu_notifier_invalidate_range_end(src_mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
> return ret;
> }
>
> @@ -1332,12 +1331,13 @@ void unmap_vmas(struct mmu_gather *tlb,
> struct vm_area_struct *vma, unsigned long start_addr,
> unsigned long end_addr)
> {
> - struct mm_struct *mm = vma->vm_mm;
> + struct mmu_notifier_range range;
>
> - mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
> + mmu_notifier_range_init(&range, vma->vm_mm, start_addr, end_addr);
> + mmu_notifier_invalidate_range_start(&range);
> for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next)
> unmap_single_vma(tlb, vma, start_addr, end_addr, NULL);
> - mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
> + mmu_notifier_invalidate_range_end(&range);
> }
>
> /**
> @@ -1351,18 +1351,18 @@ void unmap_vmas(struct mmu_gather *tlb,
> void zap_page_range(struct vm_area_struct *vma, unsigned long start,
> unsigned long size)
> {
> - struct mm_struct *mm = vma->vm_mm;
> + struct mmu_notifier_range range;
> struct mmu_gather tlb;
> - unsigned long end = start + size;
>
> lru_add_drain();
> - tlb_gather_mmu(&tlb, mm, start, end);
> - update_hiwater_rss(mm);
> - mmu_notifier_invalidate_range_start(mm, start, end);
> - for ( ; vma && vma->vm_start < end; vma = vma->vm_next)
> - unmap_single_vma(&tlb, vma, start, end, NULL);
> - mmu_notifier_invalidate_range_end(mm, start, end);
> - tlb_finish_mmu(&tlb, start, end);
> + mmu_notifier_range_init(&range, vma->vm_mm, start, start + size);
> + tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);
> + update_hiwater_rss(vma->vm_mm);
> + mmu_notifier_invalidate_range_start(&range);
> + for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next)
> + unmap_single_vma(&tlb, vma, start, range.end, NULL);
> + mmu_notifier_invalidate_range_end(&range);
> + tlb_finish_mmu(&tlb, start, range.end);
> }
>
> /**
> @@ -1377,17 +1377,17 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
> static void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
> unsigned long size, struct zap_details *details)
> {
> - struct mm_struct *mm = vma->vm_mm;
> + struct mmu_notifier_range range;
> struct mmu_gather tlb;
> - unsigned long end = address + size;
>
> lru_add_drain();
> - tlb_gather_mmu(&tlb, mm, address, end);
> - update_hiwater_rss(mm);
> - mmu_notifier_invalidate_range_start(mm, address, end);
> - unmap_single_vma(&tlb, vma, address, end, details);
> - mmu_notifier_invalidate_range_end(mm, address, end);
> - tlb_finish_mmu(&tlb, address, end);
> + mmu_notifier_range_init(&range, vma->vm_mm, address, address + size);
> + tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);
> + update_hiwater_rss(vma->vm_mm);
> + mmu_notifier_invalidate_range_start(&range);
> + unmap_single_vma(&tlb, vma, address, range.end, details);
> + mmu_notifier_invalidate_range_end(&range);
> + tlb_finish_mmu(&tlb, address, range.end);
> }
>
> /**
> @@ -2247,9 +2247,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
> struct page *new_page = NULL;
> pte_t entry;
> int page_copied = 0;
> - const unsigned long mmun_start = vmf->address & PAGE_MASK;
> - const unsigned long mmun_end = mmun_start + PAGE_SIZE;
> struct mem_cgroup *memcg;
> + struct mmu_notifier_range range;
>
> if (unlikely(anon_vma_prepare(vma)))
> goto oom;
> @@ -2272,7 +2271,9 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
>
> __SetPageUptodate(new_page);
>
> - mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, mm, vmf->address & PAGE_MASK,
> + (vmf->address & PAGE_MASK) + PAGE_SIZE);
> + mmu_notifier_invalidate_range_start(&range);
>
> /*
> * Re-check the pte - we dropped the lock
> @@ -2349,7 +2350,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
> * No need to double call mmu_notifier->invalidate_range() callback as
> * the above ptep_clear_flush_notify() did already call it.
> */
> - mmu_notifier_invalidate_range_only_end(mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_only_end(&range);
> if (old_page) {
> /*
> * Don't let another task, with possibly unlocked vma,
> @@ -4030,7 +4031,7 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
> #endif /* __PAGETABLE_PMD_FOLDED */
>
> static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
> - unsigned long *start, unsigned long *end,
> + struct mmu_notifier_range *range,
> pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
> {
> pgd_t *pgd;
> @@ -4058,10 +4059,10 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
> if (!pmdpp)
> goto out;
>
> - if (start && end) {
> - *start = address & PMD_MASK;
> - *end = *start + PMD_SIZE;
> - mmu_notifier_invalidate_range_start(mm, *start, *end);
> + if (range) {
> + mmu_notifier_range_init(range, mm, address & PMD_MASK,
> + (address & PMD_MASK) + PMD_SIZE);
> + mmu_notifier_invalidate_range_start(range);
> }
> *ptlp = pmd_lock(mm, pmd);
> if (pmd_huge(*pmd)) {
> @@ -4069,17 +4070,17 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
> return 0;
> }
> spin_unlock(*ptlp);
> - if (start && end)
> - mmu_notifier_invalidate_range_end(mm, *start, *end);
> + if (range)
> + mmu_notifier_invalidate_range_end(range);
> }
>
> if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
> goto out;
>
> - if (start && end) {
> - *start = address & PAGE_MASK;
> - *end = *start + PAGE_SIZE;
> - mmu_notifier_invalidate_range_start(mm, *start, *end);
> + if (range) {
> + range->start = address & PAGE_MASK;
> + range->end = range->start + PAGE_SIZE;
> + mmu_notifier_invalidate_range_start(range);
> }
> ptep = pte_offset_map_lock(mm, pmd, address, ptlp);
> if (!pte_present(*ptep))
> @@ -4088,8 +4089,8 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
> return 0;
> unlock:
> pte_unmap_unlock(ptep, *ptlp);
> - if (start && end)
> - mmu_notifier_invalidate_range_end(mm, *start, *end);
> + if (range)
> + mmu_notifier_invalidate_range_end(range);
> out:
> return -EINVAL;
> }
> @@ -4101,20 +4102,20 @@ static inline int follow_pte(struct mm_struct *mm, unsigned long address,
>
> /* (void) is needed to make gcc happy */
> (void) __cond_lock(*ptlp,
> - !(res = __follow_pte_pmd(mm, address, NULL, NULL,
> + !(res = __follow_pte_pmd(mm, address, NULL,
> ptepp, NULL, ptlp)));
> return res;
> }
>
> int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
> - unsigned long *start, unsigned long *end,
> + struct mmu_notifier_range *range,
> pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp)
> {
> int res;
>
> /* (void) is needed to make gcc happy */
> (void) __cond_lock(*ptlp,
> - !(res = __follow_pte_pmd(mm, address, start, end,
> + !(res = __follow_pte_pmd(mm, address, range,
> ptepp, pmdpp, ptlp)));
> return res;
> }
> diff --git a/mm/migrate.c b/mm/migrate.c
> index f7e4bfdc13b7..74f5b3208c05 100644
> --- a/mm/migrate.c
> +++ b/mm/migrate.c
> @@ -2303,6 +2303,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> */
> static void migrate_vma_collect(struct migrate_vma *migrate)
> {
> + struct mmu_notifier_range range;
> struct mm_walk mm_walk;
>
> mm_walk.pmd_entry = migrate_vma_collect_pmd;
> @@ -2314,13 +2315,11 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
> mm_walk.mm = migrate->vma->vm_mm;
> mm_walk.private = migrate;
>
> - mmu_notifier_invalidate_range_start(mm_walk.mm,
> - migrate->start,
> - migrate->end);
> + mmu_notifier_range_init(&range, mm_walk.mm, migrate->start,
> + migrate->end);
> + mmu_notifier_invalidate_range_start(&range);
> walk_page_range(migrate->start, migrate->end, &mm_walk);
> - mmu_notifier_invalidate_range_end(mm_walk.mm,
> - migrate->start,
> - migrate->end);
> + mmu_notifier_invalidate_range_end(&range);
>
> migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
> }
> @@ -2703,7 +2702,8 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
> const unsigned long start = migrate->start;
> struct vm_area_struct *vma = migrate->vma;
> struct mm_struct *mm = vma->vm_mm;
> - unsigned long addr, i, mmu_start;
> + struct mmu_notifier_range range;
> + unsigned long addr, i;
> bool notified = false;
>
> for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
> @@ -2722,11 +2722,11 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
> continue;
> }
> if (!notified) {
> - mmu_start = addr;
> notified = true;
> - mmu_notifier_invalidate_range_start(mm,
> - mmu_start,
> - migrate->end);
> +
> + mmu_notifier_range_init(&range, mm, addr,
> + migrate->end);
> + mmu_notifier_invalidate_range_start(&range);
> }
> migrate_vma_insert_page(migrate, addr, newpage,
> &migrate->src[i],
> @@ -2767,8 +2767,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
> * did already call it.
> */
> if (notified)
> - mmu_notifier_invalidate_range_only_end(mm, mmu_start,
> - migrate->end);
> + mmu_notifier_invalidate_range_only_end(&range);
> }
>
> /*
> diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
> index 5f6665ae3ee2..4c52b3514c50 100644
> --- a/mm/mmu_notifier.c
> +++ b/mm/mmu_notifier.c
> @@ -174,28 +174,20 @@ void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
> srcu_read_unlock(&srcu, id);
> }
>
> -int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> - unsigned long start, unsigned long end,
> - bool blockable)
> +int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *range)
> {
> - struct mmu_notifier_range _range, *range = &_range;
> struct mmu_notifier *mn;
> int ret = 0;
> int id;
>
> - range->blockable = blockable;
> - range->start = start;
> - range->end = end;
> - range->mm = mm;
> -
> id = srcu_read_lock(&srcu);
> - hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
> + hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {
> if (mn->ops->invalidate_range_start) {
> int _ret = mn->ops->invalidate_range_start(mn, range);
> if (_ret) {
> pr_info("%pS callback failed with %d in %sblockable context.\n",
> mn->ops->invalidate_range_start, _ret,
> - !blockable ? "non-" : "");
> + !range->blockable ? "non-" : "");
> ret = _ret;
> }
> }
> @@ -206,27 +198,14 @@ int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
> }
> EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
>
> -void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> - unsigned long start,
> - unsigned long end,
> +void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *range,
> bool only_end)
> {
> - struct mmu_notifier_range _range, *range = &_range;
> struct mmu_notifier *mn;
> int id;
>
> - /*
> - * The end call back will never be call if the start refused to go
> - * through because of blockable was false so here assume that we
> - * can block.
> - */
> - range->blockable = true;
> - range->start = start;
> - range->end = end;
> - range->mm = mm;
> -
> id = srcu_read_lock(&srcu);
> - hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
> + hlist_for_each_entry_rcu(mn, &range->mm->mmu_notifier_mm->list, hlist) {
> /*
> * Call invalidate_range here too to avoid the need for the
> * subsystem of having to register an invalidate_range_end
> @@ -241,7 +220,9 @@ void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
> * already happen under page table lock.
> */
> if (!only_end && mn->ops->invalidate_range)
> - mn->ops->invalidate_range(mn, mm, start, end);
> + mn->ops->invalidate_range(mn, range->mm,
> + range->start,
> + range->end);
> if (mn->ops->invalidate_range_end)
> mn->ops->invalidate_range_end(mn, range);
> }
> diff --git a/mm/mprotect.c b/mm/mprotect.c
> index 6d331620b9e5..36cb358db170 100644
> --- a/mm/mprotect.c
> +++ b/mm/mprotect.c
> @@ -167,11 +167,12 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
> pgprot_t newprot, int dirty_accountable, int prot_numa)
> {
> pmd_t *pmd;
> - struct mm_struct *mm = vma->vm_mm;
> unsigned long next;
> unsigned long pages = 0;
> unsigned long nr_huge_updates = 0;
> - unsigned long mni_start = 0;
> + struct mmu_notifier_range range;
> +
> + range.start = 0;
>
> pmd = pmd_offset(pud, addr);
> do {
> @@ -183,9 +184,9 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
> goto next;
>
> /* invoke the mmu notifier if the pmd is populated */
> - if (!mni_start) {
> - mni_start = addr;
> - mmu_notifier_invalidate_range_start(mm, mni_start, end);
> + if (!range.start) {
> + mmu_notifier_range_init(&range, vma->vm_mm, addr, end);
> + mmu_notifier_invalidate_range_start(&range);
> }
>
> if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
> @@ -214,8 +215,8 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
> cond_resched();
> } while (pmd++, addr = next, addr != end);
>
> - if (mni_start)
> - mmu_notifier_invalidate_range_end(mm, mni_start, end);
> + if (range.start)
> + mmu_notifier_invalidate_range_end(&range);
>
> if (nr_huge_updates)
> count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
> diff --git a/mm/mremap.c b/mm/mremap.c
> index 7f9f9180e401..def01d86e36f 100644
> --- a/mm/mremap.c
> +++ b/mm/mremap.c
> @@ -197,16 +197,14 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
> bool need_rmap_locks)
> {
> unsigned long extent, next, old_end;
> + struct mmu_notifier_range range;
> pmd_t *old_pmd, *new_pmd;
> - unsigned long mmun_start; /* For mmu_notifiers */
> - unsigned long mmun_end; /* For mmu_notifiers */
>
> old_end = old_addr + len;
> flush_cache_range(vma, old_addr, old_end);
>
> - mmun_start = old_addr;
> - mmun_end = old_end;
> - mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_range_init(&range, vma->vm_mm, old_addr, old_end);
> + mmu_notifier_invalidate_range_start(&range);
>
> for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
> cond_resched();
> @@ -247,7 +245,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
> new_pmd, new_addr, need_rmap_locks);
> }
>
> - mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
> + mmu_notifier_invalidate_range_end(&range);
>
> return len + old_addr - old_end; /* how much done */
> }
> diff --git a/mm/oom_kill.c b/mm/oom_kill.c
> index 6589f60d5018..1eea8b04f27a 100644
> --- a/mm/oom_kill.c
> +++ b/mm/oom_kill.c
> @@ -516,19 +516,20 @@ bool __oom_reap_task_mm(struct mm_struct *mm)
> * count elevated without a good reason.
> */
> if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) {
> - const unsigned long start = vma->vm_start;
> - const unsigned long end = vma->vm_end;
> + struct mmu_notifier_range range;
> struct mmu_gather tlb;
>
> - tlb_gather_mmu(&tlb, mm, start, end);
> - if (mmu_notifier_invalidate_range_start_nonblock(mm, start, end)) {
> - tlb_finish_mmu(&tlb, start, end);
> + mmu_notifier_range_init(&range, mm, vma->vm_start,
> + vma->vm_end);
> + tlb_gather_mmu(&tlb, mm, range.start, range.end);
> + if (mmu_notifier_invalidate_range_start_nonblock(&range)) {
> + tlb_finish_mmu(&tlb, range.start, range.end);
> ret = false;
> continue;
> }
> - unmap_page_range(&tlb, vma, start, end, NULL);
> - mmu_notifier_invalidate_range_end(mm, start, end);
> - tlb_finish_mmu(&tlb, start, end);
> + unmap_page_range(&tlb, vma, range.start, range.end, NULL);
> + mmu_notifier_invalidate_range_end(&range);
> + tlb_finish_mmu(&tlb, range.start, range.end);
> }
> }
>
> diff --git a/mm/rmap.c b/mm/rmap.c
> index 85b7f9423352..c75f72f6fe0e 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -889,15 +889,17 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
> .address = address,
> .flags = PVMW_SYNC,
> };
> - unsigned long start = address, end;
> + struct mmu_notifier_range range;
> int *cleaned = arg;
>
> /*
> * We have to assume the worse case ie pmd for invalidation. Note that
> * the page can not be free from this function.
> */
> - end = min(vma->vm_end, start + (PAGE_SIZE << compound_order(page)));
> - mmu_notifier_invalidate_range_start(vma->vm_mm, start, end);
> + mmu_notifier_range_init(&range, vma->vm_mm, address,
> + min(vma->vm_end, address +
> + (PAGE_SIZE << compound_order(page))));
> + mmu_notifier_invalidate_range_start(&range);
>
> while (page_vma_mapped_walk(&pvmw)) {
> unsigned long cstart;
> @@ -949,7 +951,7 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
> (*cleaned)++;
> }
>
> - mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
> + mmu_notifier_invalidate_range_end(&range);
>
> return true;
> }
> @@ -1345,7 +1347,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
> pte_t pteval;
> struct page *subpage;
> bool ret = true;
> - unsigned long start = address, end;
> + struct mmu_notifier_range range;
> enum ttu_flags flags = (enum ttu_flags)arg;
>
> /* munlock has nothing to gain from examining un-locked vmas */
> @@ -1369,15 +1371,18 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
> * Note that the page can not be free in this function as call of
> * try_to_unmap() must hold a reference on the page.
> */
> - end = min(vma->vm_end, start + (PAGE_SIZE << compound_order(page)));
> + mmu_notifier_range_init(&range, vma->vm_mm, vma->vm_start,
> + min(vma->vm_end, vma->vm_start +
> + (PAGE_SIZE << compound_order(page))));
> if (PageHuge(page)) {
> /*
> * If sharing is possible, start and end will be adjusted
> * accordingly.
> */
> - adjust_range_if_pmd_sharing_possible(vma, &start, &end);
> + adjust_range_if_pmd_sharing_possible(vma, &range.start,
> + &range.end);
> }
> - mmu_notifier_invalidate_range_start(vma->vm_mm, start, end);
> + mmu_notifier_invalidate_range_start(&range);
>
> while (page_vma_mapped_walk(&pvmw)) {
> #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
> @@ -1428,9 +1433,10 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
> * we must flush them all. start/end were
> * already adjusted above to cover this range.
> */
> - flush_cache_range(vma, start, end);
> - flush_tlb_range(vma, start, end);
> - mmu_notifier_invalidate_range(mm, start, end);
> + flush_cache_range(vma, range.start, range.end);
> + flush_tlb_range(vma, range.start, range.end);
> + mmu_notifier_invalidate_range(mm, range.start,
> + range.end);
>
> /*
> * The ref count of the PMD page was dropped
> @@ -1650,7 +1656,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
> put_page(page);
> }
>
> - mmu_notifier_invalidate_range_end(vma->vm_mm, start, end);
> + mmu_notifier_invalidate_range_end(&range);
>
> return ret;
> }
> --
> 2.17.2
>
>
--
Jan Kara <jack@xxxxxxxx>
SUSE Labs, CR
