Re: [RFC PATCH 4/7] mm: move internal core VMA manipulation functions to own file

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Thu, Jun 27, 2024 at 01:56:12PM -0400, Liam R. Howlett wrote:
> * Lorenzo Stoakes <lstoakes@xxxxxxxxx> [240627 06:39]:
> > This patch introduces vma.c and moves internal core VMA manipulation
> > functions to this file from mmap.c.
> >
> > This allows us to isolate VMA functionality in a single place such that we
> > can create userspace testing code that invokes this functionality in an
> > environment where we can implement simple unit tests of core functionality.
> >
> > This patch ensures that core VMA functionality is explicitly marked as such
> > by its presence in mm/vma.h.
> >
> > It also places the header includes required by vma.c in vma_internal.h,
> > which is simply imported by vma.c. This makes the VMA functionality
> > testable, as userland testing code can simply stub out functionality
> > as required.
>
> My initial thought on vma_internal.h would be to contain the number of
> 'helper' functions and internal structures while mm/vma.h would have the
> interface.

This is what I've done though?

>
> In this way, we could include mm/vma.h into mm/internal.h (which most
> files you've edited already has included), and any special cases
> (mmu_notifier.c, etc) would need the addition.  vma_internal.h would
> have only things needed in the vma.c file.

Happy to change it so mm/internal.h imports mm/vma_internal.h, it does make
sense for one to include theother.

>
> On testing, we could use the header guards to exclude what we wanted by
> either just #defining the right guard, or by making an entirely new
> header with a duplicate guard with the necessary stubs/functions.

Yeah this is a nice idea as mentioned on other patch, I will be doing this.

>
> >
> > Signed-off-by: Lorenzo Stoakes <lstoakes@xxxxxxxxx>
> > ---
> >  include/linux/mm.h |   35 -
> >  mm/Makefile        |    2 +-
> >  mm/gup.c           |    1 +
> >  mm/huge_memory.c   |    1 +
> >  mm/internal.h      |  227 +----
> >  mm/madvise.c       |    1 +
> >  mm/memory.c        |    1 +
> >  mm/mempolicy.c     |    1 +
> >  mm/mlock.c         |    1 +
> >  mm/mmap.c          | 1983 +++-----------------------------------------
> >  mm/mmu_notifier.c  |    2 +
> >  mm/mprotect.c      |    1 +
> >  mm/mremap.c        |    1 +
> >  mm/mseal.c         |    2 +
> >  mm/rmap.c          |    1 +
> >  mm/userfaultfd.c   |    2 +
> >  mm/vma.c           | 1766 +++++++++++++++++++++++++++++++++++++++
> >  mm/vma.h           |  356 ++++++++
> >  mm/vma_internal.h  |  143 ++++
> >  19 files changed, 2389 insertions(+), 2138 deletions(-)
> >  create mode 100644 mm/vma.c
> >  create mode 100644 mm/vma.h
> >  create mode 100644 mm/vma_internal.h
> >
> > diff --git a/include/linux/mm.h b/include/linux/mm.h
> > index e3220439cf75..31f85db029b8 100644
> > --- a/include/linux/mm.h
> > +++ b/include/linux/mm.h
> > @@ -1004,21 +1004,6 @@ struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
> >  	return mas_prev_range(&vmi->mas, 0);
> >  }
> >
> > -static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
> > -{
> > -	return vmi->mas.index;
> > -}
> > -
> > -static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
> > -{
> > -	return vmi->mas.last + 1;
> > -}
> > -static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
> > -				      unsigned long count)
> > -{
> > -	return mas_expected_entries(&vmi->mas, count);
> > -}
> > -
> >  static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
> >  			unsigned long start, unsigned long end, gfp_t gfp)
> >  {
> > @@ -2548,21 +2533,6 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma,
> >  #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
> >  					    MM_CP_UFFD_WP_RESOLVE)
> >
> > -bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
> > -bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
> > -static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
> > -{
> > -	/*
> > -	 * We want to check manually if we can change individual PTEs writable
> > -	 * if we can't do that automatically for all PTEs in a mapping. For
> > -	 * private mappings, that's always the case when we have write
> > -	 * permissions as we properly have to handle COW.
> > -	 */
> > -	if (vma->vm_flags & VM_SHARED)
> > -		return vma_wants_writenotify(vma, vma->vm_page_prot);
> > -	return !!(vma->vm_flags & VM_WRITE);
> > -
> > -}
> >  bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
> >  			     pte_t pte);
> >  extern long change_protection(struct mmu_gather *tlb,
> > @@ -3277,12 +3247,7 @@ extern int vma_expand_bottom(struct vma_iterator *vmi, struct vm_area_struct *vm
> >  			     unsigned long shift, struct vm_area_struct **next);
> >  extern int vma_shrink_top(struct vma_iterator *vmi, struct vm_area_struct *vma,
> >  			  unsigned long shift);
> > -extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
> >  extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
> > -extern void unlink_file_vma(struct vm_area_struct *);
> > -extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
> > -	unsigned long addr, unsigned long len, pgoff_t pgoff,
> > -	bool *need_rmap_locks);
> >  extern void exit_mmap(struct mm_struct *);
> >
> >  static inline int check_data_rlimit(unsigned long rlim,
> > diff --git a/mm/Makefile b/mm/Makefile
> > index d2915f8c9dc0..140a22654dde 100644
> > --- a/mm/Makefile
> > +++ b/mm/Makefile
> > @@ -37,7 +37,7 @@ mmu-y			:= nommu.o
> >  mmu-$(CONFIG_MMU)	:= highmem.o memory.o mincore.o \
> >  			   mlock.o mmap.o mmu_gather.o mprotect.o mremap.o \
> >  			   msync.o page_vma_mapped.o pagewalk.o \
> > -			   pgtable-generic.o rmap.o vmalloc.o
> > +			   pgtable-generic.o rmap.o vmalloc.o vma.o
> >
> >
> >  ifdef CONFIG_CROSS_MEMORY_ATTACH
> > diff --git a/mm/gup.c b/mm/gup.c
> > index 8bea9ad80984..34b846352679 100644
> > --- a/mm/gup.c
> > +++ b/mm/gup.c
> > @@ -26,6 +26,7 @@
> >  #include <asm/tlbflush.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  struct follow_page_context {
> >  	struct dev_pagemap *pgmap;
> > diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> > index c7ce28f6b7f3..de6f150ed97b 100644
> > --- a/mm/huge_memory.c
> > +++ b/mm/huge_memory.c
> > @@ -44,6 +44,7 @@
> >  #include <asm/tlb.h>
> >  #include <asm/pgalloc.h>
> >  #include "internal.h"
> > +#include "vma.h"
> >  #include "swap.h"
> >
> >  #define CREATE_TRACE_POINTS
> > diff --git a/mm/internal.h b/mm/internal.h
> > index f7779727bb78..76b4821cd751 100644
> > --- a/mm/internal.h
> > +++ b/mm/internal.h
> > @@ -8,7 +8,9 @@
> >  #define __MM_INTERNAL_H
> >
> >  #include <linux/fs.h>
> > +#include <linux/khugepaged.h>
> >  #include <linux/mm.h>
> > +#include <linux/mm_inline.h>
> >  #include <linux/pagemap.h>
> >  #include <linux/rmap.h>
> >  #include <linux/swap.h>
> > @@ -778,37 +780,6 @@ static inline bool free_area_empty(struct free_area *area, int migratetype)
> >  	return list_empty(&area->free_list[migratetype]);
> >  }
> >
> > -/*
> > - * These three helpers classifies VMAs for virtual memory accounting.
> > - */
> > -
> > -/*
> > - * Executable code area - executable, not writable, not stack
> > - */
> > -static inline bool is_exec_mapping(vm_flags_t flags)
> > -{
> > -	return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
> > -}
> > -
> > -/*
> > - * Stack area (including shadow stacks)
> > - *
> > - * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
> > - * do_mmap() forbids all other combinations.
> > - */
> > -static inline bool is_stack_mapping(vm_flags_t flags)
> > -{
> > -	return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
> > -}
> > -
> > -/*
> > - * Data area - private, writable, not stack
> > - */
> > -static inline bool is_data_mapping(vm_flags_t flags)
> > -{
> > -	return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
> > -}
> > -
> >  /* mm/util.c */
> >  struct anon_vma *folio_anon_vma(struct folio *folio);
> >
> > @@ -1237,80 +1208,6 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr,
> >  void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
> >  	       pmd_t *pmd, bool write);
> >
> > -/*
> > - * mm/mmap.c
> > - */
> > -struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> > -					struct vm_area_struct *vma,
> > -					unsigned long delta);
> > -
> > -struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> > -				  struct vm_area_struct *prev,
> > -				  struct vm_area_struct *vma,
> > -				  unsigned long start, unsigned long end,
> > -				  unsigned long vm_flags,
> > -				  struct mempolicy *policy,
> > -				  struct vm_userfaultfd_ctx uffd_ctx,
> > -				  struct anon_vma_name *anon_name);
> > -
> > -/* We are about to modify the VMA's flags. */
> > -static inline struct vm_area_struct
> > -*vma_modify_flags(struct vma_iterator *vmi,
> > -		  struct vm_area_struct *prev,
> > -		  struct vm_area_struct *vma,
> > -		  unsigned long start, unsigned long end,
> > -		  unsigned long new_flags)
> > -{
> > -	return vma_modify(vmi, prev, vma, start, end, new_flags,
> > -			  vma_policy(vma), vma->vm_userfaultfd_ctx,
> > -			  anon_vma_name(vma));
> > -}
> > -
> > -/* We are about to modify the VMA's flags and/or anon_name. */
> > -static inline struct vm_area_struct
> > -*vma_modify_flags_name(struct vma_iterator *vmi,
> > -		       struct vm_area_struct *prev,
> > -		       struct vm_area_struct *vma,
> > -		       unsigned long start,
> > -		       unsigned long end,
> > -		       unsigned long new_flags,
> > -		       struct anon_vma_name *new_name)
> > -{
> > -	return vma_modify(vmi, prev, vma, start, end, new_flags,
> > -			  vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
> > -}
> > -
> > -/* We are about to modify the VMA's memory policy. */
> > -static inline struct vm_area_struct
> > -*vma_modify_policy(struct vma_iterator *vmi,
> > -		   struct vm_area_struct *prev,
> > -		   struct vm_area_struct *vma,
> > -		   unsigned long start, unsigned long end,
> > -		   struct mempolicy *new_pol)
> > -{
> > -	return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
> > -			  new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> > -}
> > -
> > -/* We are about to modify the VMA's flags and/or uffd context. */
> > -static inline struct vm_area_struct
> > -*vma_modify_flags_uffd(struct vma_iterator *vmi,
> > -		       struct vm_area_struct *prev,
> > -		       struct vm_area_struct *vma,
> > -		       unsigned long start, unsigned long end,
> > -		       unsigned long new_flags,
> > -		       struct vm_userfaultfd_ctx new_ctx)
> > -{
> > -	return vma_modify(vmi, prev, vma, start, end, new_flags,
> > -			  vma_policy(vma), new_ctx, anon_vma_name(vma));
> > -}
> > -
> > -int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -	       unsigned long start, unsigned long end, pgoff_t pgoff,
> > -		      struct vm_area_struct *next);
> > -int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -	       unsigned long start, unsigned long end, pgoff_t pgoff);
> > -
> >  enum {
> >  	/* mark page accessed */
> >  	FOLL_TOUCH = 1 << 16,
> > @@ -1437,117 +1334,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte
> >  	return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);
> >  }
> >
> > -static inline void vma_iter_config(struct vma_iterator *vmi,
> > -		unsigned long index, unsigned long last)
> > -{
> > -	__mas_set_range(&vmi->mas, index, last - 1);
> > -}
> > -
> > -static inline void vma_iter_reset(struct vma_iterator *vmi)
> > -{
> > -	mas_reset(&vmi->mas);
> > -}
> > -
> > -static inline
> > -struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
> > -{
> > -	return mas_prev_range(&vmi->mas, min);
> > -}
> > -
> > -static inline
> > -struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
> > -{
> > -	return mas_next_range(&vmi->mas, max);
> > -}
> > -
> > -static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
> > -				       unsigned long max, unsigned long size)
> > -{
> > -	return mas_empty_area(&vmi->mas, min, max - 1, size);
> > -}
> > -
> > -static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
> > -					unsigned long max, unsigned long size)
> > -{
> > -	return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
> > -}
> > -
> > -/*
> > - * VMA Iterator functions shared between nommu and mmap
> > - */
> > -static inline int vma_iter_prealloc(struct vma_iterator *vmi,
> > -		struct vm_area_struct *vma)
> > -{
> > -	return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
> > -}
> > -
> > -static inline void vma_iter_clear(struct vma_iterator *vmi)
> > -{
> > -	mas_store_prealloc(&vmi->mas, NULL);
> > -}
> > -
> > -static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
> > -{
> > -	return mas_walk(&vmi->mas);
> > -}
> > -
> > -/* Store a VMA with preallocated memory */
> > -static inline void vma_iter_store(struct vma_iterator *vmi,
> > -				  struct vm_area_struct *vma)
> > -{
> > -
> > -#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> > -	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> > -			vmi->mas.index > vma->vm_start)) {
> > -		pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
> > -			vmi->mas.index, vma->vm_start, vma->vm_start,
> > -			vma->vm_end, vmi->mas.index, vmi->mas.last);
> > -	}
> > -	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> > -			vmi->mas.last <  vma->vm_start)) {
> > -		pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
> > -		       vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
> > -		       vmi->mas.index, vmi->mas.last);
> > -	}
> > -#endif
> > -
> > -	if (vmi->mas.status != ma_start &&
> > -	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> > -		vma_iter_invalidate(vmi);
> > -
> > -	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> > -	mas_store_prealloc(&vmi->mas, vma);
> > -}
> > -
> > -static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
> > -			struct vm_area_struct *vma, gfp_t gfp)
> > -{
> > -	if (vmi->mas.status != ma_start &&
> > -	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> > -		vma_iter_invalidate(vmi);
> > -
> > -	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> > -	mas_store_gfp(&vmi->mas, vma, gfp);
> > -	if (unlikely(mas_is_err(&vmi->mas)))
> > -		return -ENOMEM;
> > -
> > -	return 0;
> > -}
> > -
> > -/*
> > - * VMA lock generalization
> > - */
> > -struct vma_prepare {
> > -	struct vm_area_struct *vma;
> > -	struct vm_area_struct *adj_next;
> > -	struct file *file;
> > -	struct address_space *mapping;
> > -	struct anon_vma *anon_vma;
> > -	struct vm_area_struct *insert;
> > -	struct vm_area_struct *remove;
> > -	struct vm_area_struct *remove2;
> > -};
> > -
> >  void __meminit __init_single_page(struct page *page, unsigned long pfn,
> >  				unsigned long zone, int nid);
> >
> > @@ -1636,13 +1422,4 @@ static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry,
> >  void workingset_update_node(struct xa_node *node);
> >  extern struct list_lru shadow_nodes;
> >
> > -struct unlink_vma_file_batch {
> > -	int count;
> > -	struct vm_area_struct *vmas[8];
> > -};
> > -
> > -void unlink_file_vma_batch_init(struct unlink_vma_file_batch *);
> > -void unlink_file_vma_batch_add(struct unlink_vma_file_batch *, struct vm_area_struct *);
> > -void unlink_file_vma_batch_final(struct unlink_vma_file_batch *);
> > -
> >  #endif	/* __MM_INTERNAL_H */
> > diff --git a/mm/madvise.c b/mm/madvise.c
> > index 96c026fe0c99..42f62a8efd71 100644
> > --- a/mm/madvise.c
> > +++ b/mm/madvise.c
> > @@ -35,6 +35,7 @@
> >  #include <asm/tlb.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >  #include "swap.h"
> >
> >  struct madvise_walk_private {
> > diff --git a/mm/memory.c b/mm/memory.c
> > index 0a769f34bbb2..a2ca7df9d2cf 100644
> > --- a/mm/memory.c
> > +++ b/mm/memory.c
> > @@ -90,6 +90,7 @@
> >
> >  #include "pgalloc-track.h"
> >  #include "internal.h"
> > +#include "vma.h"
> >  #include "swap.h"
> >
> >  #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
> > diff --git a/mm/mempolicy.c b/mm/mempolicy.c
> > index f73acb01ad45..3dad2b52f319 100644
> > --- a/mm/mempolicy.c
> > +++ b/mm/mempolicy.c
> > @@ -115,6 +115,7 @@
> >  #include <linux/uaccess.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  /* Internal flags */
> >  #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
> > diff --git a/mm/mlock.c b/mm/mlock.c
> > index 52d6e401ad67..ac84378bb796 100644
> > --- a/mm/mlock.c
> > +++ b/mm/mlock.c
> > @@ -27,6 +27,7 @@
> >  #include <linux/secretmem.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  struct mlock_fbatch {
> >  	local_lock_t lock;
> > diff --git a/mm/mmap.c b/mm/mmap.c
> > index 574e69a04ebe..b4f7c1ea3f0f 100644
> > --- a/mm/mmap.c
> > +++ b/mm/mmap.c
> > @@ -57,6 +57,7 @@
> >  #include <trace/events/mmap.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  #ifndef arch_mmap_check
> >  #define arch_mmap_check(addr, len, flags)	(0)
> > @@ -76,16 +77,6 @@ int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
> >  static bool ignore_rlimit_data;
> >  core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
> >
> > -static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> > -		struct vm_area_struct *vma, struct vm_area_struct *prev,
> > -		struct vm_area_struct *next, unsigned long start,
> > -		unsigned long end, unsigned long tree_end, bool mm_wr_locked);
> > -
> > -static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
> > -{
> > -	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
> > -}
> > -
> >  /* Update vma->vm_page_prot to reflect vma->vm_flags. */
> >  void vma_set_page_prot(struct vm_area_struct *vma)
> >  {
> > @@ -101,100 +92,6 @@ void vma_set_page_prot(struct vm_area_struct *vma)
> >  	WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
> >  }
> >
> > -/*
> > - * Requires inode->i_mapping->i_mmap_rwsem
> > - */
> > -static void __remove_shared_vm_struct(struct vm_area_struct *vma,
> > -				      struct address_space *mapping)
> > -{
> > -	if (vma_is_shared_maywrite(vma))
> > -		mapping_unmap_writable(mapping);
> > -
> > -	flush_dcache_mmap_lock(mapping);
> > -	vma_interval_tree_remove(vma, &mapping->i_mmap);
> > -	flush_dcache_mmap_unlock(mapping);
> > -}
> > -
> > -/*
> > - * Unlink a file-based vm structure from its interval tree, to hide
> > - * vma from rmap and vmtruncate before freeing its page tables.
> > - */
> > -void unlink_file_vma(struct vm_area_struct *vma)
> > -{
> > -	struct file *file = vma->vm_file;
> > -
> > -	if (file) {
> > -		struct address_space *mapping = file->f_mapping;
> > -		i_mmap_lock_write(mapping);
> > -		__remove_shared_vm_struct(vma, mapping);
> > -		i_mmap_unlock_write(mapping);
> > -	}
> > -}
> > -
> > -void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)
> > -{
> > -	vb->count = 0;
> > -}
> > -
> > -static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb)
> > -{
> > -	struct address_space *mapping;
> > -	int i;
> > -
> > -	mapping = vb->vmas[0]->vm_file->f_mapping;
> > -	i_mmap_lock_write(mapping);
> > -	for (i = 0; i < vb->count; i++) {
> > -		VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping);
> > -		__remove_shared_vm_struct(vb->vmas[i], mapping);
> > -	}
> > -	i_mmap_unlock_write(mapping);
> > -
> > -	unlink_file_vma_batch_init(vb);
> > -}
> > -
> > -void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
> > -			       struct vm_area_struct *vma)
> > -{
> > -	if (vma->vm_file == NULL)
> > -		return;
> > -
> > -	if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) ||
> > -	    vb->count == ARRAY_SIZE(vb->vmas))
> > -		unlink_file_vma_batch_process(vb);
> > -
> > -	vb->vmas[vb->count] = vma;
> > -	vb->count++;
> > -}
> > -
> > -void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb)
> > -{
> > -	if (vb->count > 0)
> > -		unlink_file_vma_batch_process(vb);
> > -}
> > -
> > -/*
> > - * Close a vm structure and free it.
> > - */
> > -static void remove_vma(struct vm_area_struct *vma, bool unreachable)
> > -{
> > -	might_sleep();
> > -	if (vma->vm_ops && vma->vm_ops->close)
> > -		vma->vm_ops->close(vma);
> > -	if (vma->vm_file)
> > -		fput(vma->vm_file);
> > -	mpol_put(vma_policy(vma));
> > -	if (unreachable)
> > -		__vm_area_free(vma);
> > -	else
> > -		vm_area_free(vma);
> > -}
> > -
> > -static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
> > -						    unsigned long min)
> > -{
> > -	return mas_prev(&vmi->mas, min);
> > -}
> > -
> >  /*
> >   * check_brk_limits() - Use platform specific check of range & verify mlock
> >   * limits.
> > @@ -298,893 +195,24 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
> >  	brkvma = vma_prev_limit(&vmi, mm->start_brk);
> >  	/* Ok, looks good - let it rip. */
> >  	if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
> > -		goto out;
> > -
> > -	mm->brk = brk;
> > -	if (mm->def_flags & VM_LOCKED)
> > -		populate = true;
> > -
> > -success:
> > -	mmap_write_unlock(mm);
> > -success_unlocked:
> > -	userfaultfd_unmap_complete(mm, &uf);
> > -	if (populate)
> > -		mm_populate(oldbrk, newbrk - oldbrk);
> > -	return brk;
> > -
> > -out:
> > -	mm->brk = origbrk;
> > -	mmap_write_unlock(mm);
> > -	return origbrk;
> > -}
> > -
> > -#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> > -static void validate_mm(struct mm_struct *mm)
> > -{
> > -	int bug = 0;
> > -	int i = 0;
> > -	struct vm_area_struct *vma;
> > -	VMA_ITERATOR(vmi, mm, 0);
> > -
> > -	mt_validate(&mm->mm_mt);
> > -	for_each_vma(vmi, vma) {
> > -#ifdef CONFIG_DEBUG_VM_RB
> > -		struct anon_vma *anon_vma = vma->anon_vma;
> > -		struct anon_vma_chain *avc;
> > -#endif
> > -		unsigned long vmi_start, vmi_end;
> > -		bool warn = 0;
> > -
> > -		vmi_start = vma_iter_addr(&vmi);
> > -		vmi_end = vma_iter_end(&vmi);
> > -		if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
> > -			warn = 1;
> > -
> > -		if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
> > -			warn = 1;
> > -
> > -		if (warn) {
> > -			pr_emerg("issue in %s\n", current->comm);
> > -			dump_stack();
> > -			dump_vma(vma);
> > -			pr_emerg("tree range: %px start %lx end %lx\n", vma,
> > -				 vmi_start, vmi_end - 1);
> > -			vma_iter_dump_tree(&vmi);
> > -		}
> > -
> > -#ifdef CONFIG_DEBUG_VM_RB
> > -		if (anon_vma) {
> > -			anon_vma_lock_read(anon_vma);
> > -			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > -				anon_vma_interval_tree_verify(avc);
> > -			anon_vma_unlock_read(anon_vma);
> > -		}
> > -#endif
> > -		i++;
> > -	}
> > -	if (i != mm->map_count) {
> > -		pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
> > -		bug = 1;
> > -	}
> > -	VM_BUG_ON_MM(bug, mm);
> > -}
> > -
> > -#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */
> > -#define validate_mm(mm) do { } while (0)
> > -#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
> > -
> > -/*
> > - * vma has some anon_vma assigned, and is already inserted on that
> > - * anon_vma's interval trees.
> > - *
> > - * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
> > - * vma must be removed from the anon_vma's interval trees using
> > - * anon_vma_interval_tree_pre_update_vma().
> > - *
> > - * After the update, the vma will be reinserted using
> > - * anon_vma_interval_tree_post_update_vma().
> > - *
> > - * The entire update must be protected by exclusive mmap_lock and by
> > - * the root anon_vma's mutex.
> > - */
> > -static inline void
> > -anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
> > -{
> > -	struct anon_vma_chain *avc;
> > -
> > -	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > -		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
> > -}
> > -
> > -static inline void
> > -anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
> > -{
> > -	struct anon_vma_chain *avc;
> > -
> > -	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > -		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
> > -}
> > -
> > -static unsigned long count_vma_pages_range(struct mm_struct *mm,
> > -		unsigned long addr, unsigned long end)
> > -{
> > -	VMA_ITERATOR(vmi, mm, addr);
> > -	struct vm_area_struct *vma;
> > -	unsigned long nr_pages = 0;
> > -
> > -	for_each_vma_range(vmi, vma, end) {
> > -		unsigned long vm_start = max(addr, vma->vm_start);
> > -		unsigned long vm_end = min(end, vma->vm_end);
> > -
> > -		nr_pages += PHYS_PFN(vm_end - vm_start);
> > -	}
> > -
> > -	return nr_pages;
> > -}
> > -
> > -static void __vma_link_file(struct vm_area_struct *vma,
> > -			    struct address_space *mapping)
> > -{
> > -	if (vma_is_shared_maywrite(vma))
> > -		mapping_allow_writable(mapping);
> > -
> > -	flush_dcache_mmap_lock(mapping);
> > -	vma_interval_tree_insert(vma, &mapping->i_mmap);
> > -	flush_dcache_mmap_unlock(mapping);
> > -}
> > -
> > -static void vma_link_file(struct vm_area_struct *vma)
> > -{
> > -	struct file *file = vma->vm_file;
> > -	struct address_space *mapping;
> > -
> > -	if (file) {
> > -		mapping = file->f_mapping;
> > -		i_mmap_lock_write(mapping);
> > -		__vma_link_file(vma, mapping);
> > -		i_mmap_unlock_write(mapping);
> > -	}
> > -}
> > -
> > -static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
> > -{
> > -	VMA_ITERATOR(vmi, mm, 0);
> > -
> > -	vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
> > -	if (vma_iter_prealloc(&vmi, vma))
> > -		return -ENOMEM;
> > -
> > -	vma_start_write(vma);
> > -	vma_iter_store(&vmi, vma);
> > -	vma_link_file(vma);
> > -	mm->map_count++;
> > -	validate_mm(mm);
> > -	return 0;
> > -}
> > -
> > -/*
> > - * init_multi_vma_prep() - Initializer for struct vma_prepare
> > - * @vp: The vma_prepare struct
> > - * @vma: The vma that will be altered once locked
> > - * @next: The next vma if it is to be adjusted
> > - * @remove: The first vma to be removed
> > - * @remove2: The second vma to be removed
> > - */
> > -static inline void init_multi_vma_prep(struct vma_prepare *vp,
> > -		struct vm_area_struct *vma, struct vm_area_struct *next,
> > -		struct vm_area_struct *remove, struct vm_area_struct *remove2)
> > -{
> > -	memset(vp, 0, sizeof(struct vma_prepare));
> > -	vp->vma = vma;
> > -	vp->anon_vma = vma->anon_vma;
> > -	vp->remove = remove;
> > -	vp->remove2 = remove2;
> > -	vp->adj_next = next;
> > -	if (!vp->anon_vma && next)
> > -		vp->anon_vma = next->anon_vma;
> > -
> > -	vp->file = vma->vm_file;
> > -	if (vp->file)
> > -		vp->mapping = vma->vm_file->f_mapping;
> > -
> > -}
> > -
> > -/*
> > - * init_vma_prep() - Initializer wrapper for vma_prepare struct
> > - * @vp: The vma_prepare struct
> > - * @vma: The vma that will be altered once locked
> > - */
> > -static inline void init_vma_prep(struct vma_prepare *vp,
> > -				 struct vm_area_struct *vma)
> > -{
> > -	init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
> > -}
> > -
> > -
> > -/*
> > - * vma_prepare() - Helper function for handling locking VMAs prior to altering
> > - * @vp: The initialized vma_prepare struct
> > - */
> > -static inline void vma_prepare(struct vma_prepare *vp)
> > -{
> > -	if (vp->file) {
> > -		uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
> > -
> > -		if (vp->adj_next)
> > -			uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
> > -				      vp->adj_next->vm_end);
> > -
> > -		i_mmap_lock_write(vp->mapping);
> > -		if (vp->insert && vp->insert->vm_file) {
> > -			/*
> > -			 * Put into interval tree now, so instantiated pages
> > -			 * are visible to arm/parisc __flush_dcache_page
> > -			 * throughout; but we cannot insert into address
> > -			 * space until vma start or end is updated.
> > -			 */
> > -			__vma_link_file(vp->insert,
> > -					vp->insert->vm_file->f_mapping);
> > -		}
> > -	}
> > -
> > -	if (vp->anon_vma) {
> > -		anon_vma_lock_write(vp->anon_vma);
> > -		anon_vma_interval_tree_pre_update_vma(vp->vma);
> > -		if (vp->adj_next)
> > -			anon_vma_interval_tree_pre_update_vma(vp->adj_next);
> > -	}
> > -
> > -	if (vp->file) {
> > -		flush_dcache_mmap_lock(vp->mapping);
> > -		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
> > -		if (vp->adj_next)
> > -			vma_interval_tree_remove(vp->adj_next,
> > -						 &vp->mapping->i_mmap);
> > -	}
> > -
> > -}
> > -
> > -/*
> > - * vma_complete- Helper function for handling the unlocking after altering VMAs,
> > - * or for inserting a VMA.
> > - *
> > - * @vp: The vma_prepare struct
> > - * @vmi: The vma iterator
> > - * @mm: The mm_struct
> > - */
> > -static inline void vma_complete(struct vma_prepare *vp,
> > -				struct vma_iterator *vmi, struct mm_struct *mm)
> > -{
> > -	if (vp->file) {
> > -		if (vp->adj_next)
> > -			vma_interval_tree_insert(vp->adj_next,
> > -						 &vp->mapping->i_mmap);
> > -		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
> > -		flush_dcache_mmap_unlock(vp->mapping);
> > -	}
> > -
> > -	if (vp->remove && vp->file) {
> > -		__remove_shared_vm_struct(vp->remove, vp->mapping);
> > -		if (vp->remove2)
> > -			__remove_shared_vm_struct(vp->remove2, vp->mapping);
> > -	} else if (vp->insert) {
> > -		/*
> > -		 * split_vma has split insert from vma, and needs
> > -		 * us to insert it before dropping the locks
> > -		 * (it may either follow vma or precede it).
> > -		 */
> > -		vma_iter_store(vmi, vp->insert);
> > -		mm->map_count++;
> > -	}
> > -
> > -	if (vp->anon_vma) {
> > -		anon_vma_interval_tree_post_update_vma(vp->vma);
> > -		if (vp->adj_next)
> > -			anon_vma_interval_tree_post_update_vma(vp->adj_next);
> > -		anon_vma_unlock_write(vp->anon_vma);
> > -	}
> > -
> > -	if (vp->file) {
> > -		i_mmap_unlock_write(vp->mapping);
> > -		uprobe_mmap(vp->vma);
> > -
> > -		if (vp->adj_next)
> > -			uprobe_mmap(vp->adj_next);
> > -	}
> > -
> > -	if (vp->remove) {
> > -again:
> > -		vma_mark_detached(vp->remove, true);
> > -		if (vp->file) {
> > -			uprobe_munmap(vp->remove, vp->remove->vm_start,
> > -				      vp->remove->vm_end);
> > -			fput(vp->file);
> > -		}
> > -		if (vp->remove->anon_vma)
> > -			anon_vma_merge(vp->vma, vp->remove);
> > -		mm->map_count--;
> > -		mpol_put(vma_policy(vp->remove));
> > -		if (!vp->remove2)
> > -			WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
> > -		vm_area_free(vp->remove);
> > -
> > -		/*
> > -		 * In mprotect's case 6 (see comments on vma_merge),
> > -		 * we are removing both mid and next vmas
> > -		 */
> > -		if (vp->remove2) {
> > -			vp->remove = vp->remove2;
> > -			vp->remove2 = NULL;
> > -			goto again;
> > -		}
> > -	}
> > -	if (vp->insert && vp->file)
> > -		uprobe_mmap(vp->insert);
> > -	validate_mm(mm);
> > -}
> > -
> > -/*
> > - * dup_anon_vma() - Helper function to duplicate anon_vma
> > - * @dst: The destination VMA
> > - * @src: The source VMA
> > - * @dup: Pointer to the destination VMA when successful.
> > - *
> > - * Returns: 0 on success.
> > - */
> > -static inline int dup_anon_vma(struct vm_area_struct *dst,
> > -		struct vm_area_struct *src, struct vm_area_struct **dup)
> > -{
> > -	/*
> > -	 * Easily overlooked: when mprotect shifts the boundary, make sure the
> > -	 * expanding vma has anon_vma set if the shrinking vma had, to cover any
> > -	 * anon pages imported.
> > -	 */
> > -	if (src->anon_vma && !dst->anon_vma) {
> > -		int ret;
> > -
> > -		vma_assert_write_locked(dst);
> > -		dst->anon_vma = src->anon_vma;
> > -		ret = anon_vma_clone(dst, src);
> > -		if (ret)
> > -			return ret;
> > -
> > -		*dup = dst;
> > -	}
> > -
> > -	return 0;
> > -}
> > -
> > -/*
> > - * vma_expand - Expand an existing VMA
> > - *
> > - * @vmi: The vma iterator
> > - * @vma: The vma to expand
> > - * @start: The start of the vma
> > - * @end: The exclusive end of the vma
> > - * @pgoff: The page offset of vma
> > - * @next: The current of next vma.
> > - *
> > - * Expand @vma to @start and @end.  Can expand off the start and end.  Will
> > - * expand over @next if it's different from @vma and @end == @next->vm_end.
> > - * Checking if the @vma can expand and merge with @next needs to be handled by
> > - * the caller.
> > - *
> > - * Returns: 0 on success
> > - */
> > -int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -	       unsigned long start, unsigned long end, pgoff_t pgoff,
> > -	       struct vm_area_struct *next)
> > -{
> > -	struct vm_area_struct *anon_dup = NULL;
> > -	bool remove_next = false;
> > -	struct vma_prepare vp;
> > -
> > -	vma_start_write(vma);
> > -	if (next && (vma != next) && (end == next->vm_end)) {
> > -		int ret;
> > -
> > -		remove_next = true;
> > -		vma_start_write(next);
> > -		ret = dup_anon_vma(vma, next, &anon_dup);
> > -		if (ret)
> > -			return ret;
> > -	}
> > -
> > -	init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL);
> > -	/* Not merging but overwriting any part of next is not handled. */
> > -	VM_WARN_ON(next && !vp.remove &&
> > -		  next != vma && end > next->vm_start);
> > -	/* Only handles expanding */
> > -	VM_WARN_ON(vma->vm_start < start || vma->vm_end > end);
> > -
> > -	/* Note: vma iterator must be pointing to 'start' */
> > -	vma_iter_config(vmi, start, end);
> > -	if (vma_iter_prealloc(vmi, vma))
> > -		goto nomem;
> > -
> > -	vma_prepare(&vp);
> > -	vma_adjust_trans_huge(vma, start, end, 0);
> > -	vma_set_range(vma, start, end, pgoff);
> > -	vma_iter_store(vmi, vma);
> > -
> > -	vma_complete(&vp, vmi, vma->vm_mm);
> > -	return 0;
> > -
> > -nomem:
> > -	if (anon_dup)
> > -		unlink_anon_vmas(anon_dup);
> > -	return -ENOMEM;
> > -}
> > -
> > -/*
> > - * vma_shrink() - Reduce an existing VMAs memory area
> > - * @vmi: The vma iterator
> > - * @vma: The VMA to modify
> > - * @start: The new start
> > - * @end: The new end
> > - *
> > - * Returns: 0 on success, -ENOMEM otherwise
> > - */
> > -int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -	       unsigned long start, unsigned long end, pgoff_t pgoff)
> > -{
> > -	struct vma_prepare vp;
> > -
> > -	WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
> > -
> > -	if (vma->vm_start < start)
> > -		vma_iter_config(vmi, vma->vm_start, start);
> > -	else
> > -		vma_iter_config(vmi, end, vma->vm_end);
> > -
> > -	if (vma_iter_prealloc(vmi, NULL))
> > -		return -ENOMEM;
> > -
> > -	vma_start_write(vma);
> > -
> > -	init_vma_prep(&vp, vma);
> > -	vma_prepare(&vp);
> > -	vma_adjust_trans_huge(vma, start, end, 0);
> > -
> > -	vma_iter_clear(vmi);
> > -	vma_set_range(vma, start, end, pgoff);
> > -	vma_complete(&vp, vmi, vma->vm_mm);
> > -	return 0;
> > -}
> > -
> > -/*
> > - * If the vma has a ->close operation then the driver probably needs to release
> > - * per-vma resources, so we don't attempt to merge those if the caller indicates
> > - * the current vma may be removed as part of the merge.
> > - */
> > -static inline bool is_mergeable_vma(struct vm_area_struct *vma,
> > -		struct file *file, unsigned long vm_flags,
> > -		struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > -		struct anon_vma_name *anon_name, bool may_remove_vma)
> > -{
> > -	/*
> > -	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
> > -	 * match the flags but dirty bit -- the caller should mark
> > -	 * merged VMA as dirty. If dirty bit won't be excluded from
> > -	 * comparison, we increase pressure on the memory system forcing
> > -	 * the kernel to generate new VMAs when old one could be
> > -	 * extended instead.
> > -	 */
> > -	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
> > -		return false;
> > -	if (vma->vm_file != file)
> > -		return false;
> > -	if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
> > -		return false;
> > -	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
> > -		return false;
> > -	if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
> > -		return false;
> > -	return true;
> > -}
> > -
> > -static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
> > -		 struct anon_vma *anon_vma2, struct vm_area_struct *vma)
> > -{
> > -	/*
> > -	 * The list_is_singular() test is to avoid merging VMA cloned from
> > -	 * parents. This can improve scalability caused by anon_vma lock.
> > -	 */
> > -	if ((!anon_vma1 || !anon_vma2) && (!vma ||
> > -		list_is_singular(&vma->anon_vma_chain)))
> > -		return true;
> > -	return anon_vma1 == anon_vma2;
> > -}
> > -
> > -/*
> > - * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> > - * in front of (at a lower virtual address and file offset than) the vma.
> > - *
> > - * We cannot merge two vmas if they have differently assigned (non-NULL)
> > - * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> > - *
> > - * We don't check here for the merged mmap wrapping around the end of pagecache
> > - * indices (16TB on ia32) because do_mmap() does not permit mmap's which
> > - * wrap, nor mmaps which cover the final page at index -1UL.
> > - *
> > - * We assume the vma may be removed as part of the merge.
> > - */
> > -static bool
> > -can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
> > -		struct anon_vma *anon_vma, struct file *file,
> > -		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > -		struct anon_vma_name *anon_name)
> > -{
> > -	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&
> > -	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> > -		if (vma->vm_pgoff == vm_pgoff)
> > -			return true;
> > -	}
> > -	return false;
> > -}
> > -
> > -/*
> > - * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> > - * beyond (at a higher virtual address and file offset than) the vma.
> > - *
> > - * We cannot merge two vmas if they have differently assigned (non-NULL)
> > - * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> > - *
> > - * We assume that vma is not removed as part of the merge.
> > - */
> > -static bool
> > -can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
> > -		struct anon_vma *anon_vma, struct file *file,
> > -		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > -		struct anon_vma_name *anon_name)
> > -{
> > -	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&
> > -	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> > -		pgoff_t vm_pglen;
> > -		vm_pglen = vma_pages(vma);
> > -		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
> > -			return true;
> > -	}
> > -	return false;
> > -}
> > -
> > -/*
> > - * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
> > - * figure out whether that can be merged with its predecessor or its
> > - * successor.  Or both (it neatly fills a hole).
> > - *
> > - * In most cases - when called for mmap, brk or mremap - [addr,end) is
> > - * certain not to be mapped by the time vma_merge is called; but when
> > - * called for mprotect, it is certain to be already mapped (either at
> > - * an offset within prev, or at the start of next), and the flags of
> > - * this area are about to be changed to vm_flags - and the no-change
> > - * case has already been eliminated.
> > - *
> > - * The following mprotect cases have to be considered, where **** is
> > - * the area passed down from mprotect_fixup, never extending beyond one
> > - * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts
> > - * at the same address as **** and is of the same or larger span, and
> > - * NNNN the next vma after ****:
> > - *
> > - *     ****             ****                   ****
> > - *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPCCCCCC
> > - *    cannot merge    might become       might become
> > - *                    PPNNNNNNNNNN       PPPPPPPPPPCC
> > - *    mmap, brk or    case 4 below       case 5 below
> > - *    mremap move:
> > - *                        ****               ****
> > - *                    PPPP    NNNN       PPPPCCCCNNNN
> > - *                    might become       might become
> > - *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
> > - *                    PPPPPPPPNNNN 2 or  PPPPPPPPNNNN 7 or
> > - *                    PPPPNNNNNNNN 3     PPPPNNNNNNNN 8
> > - *
> > - * It is important for case 8 that the vma CCCC overlapping the
> > - * region **** is never going to extended over NNNN. Instead NNNN must
> > - * be extended in region **** and CCCC must be removed. This way in
> > - * all cases where vma_merge succeeds, the moment vma_merge drops the
> > - * rmap_locks, the properties of the merged vma will be already
> > - * correct for the whole merged range. Some of those properties like
> > - * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
> > - * be correct for the whole merged range immediately after the
> > - * rmap_locks are released. Otherwise if NNNN would be removed and
> > - * CCCC would be extended over the NNNN range, remove_migration_ptes
> > - * or other rmap walkers (if working on addresses beyond the "end"
> > - * parameter) may establish ptes with the wrong permissions of CCCC
> > - * instead of the right permissions of NNNN.
> > - *
> > - * In the code below:
> > - * PPPP is represented by *prev
> > - * CCCC is represented by *curr or not represented at all (NULL)
> > - * NNNN is represented by *next or not represented at all (NULL)
> > - * **** is not represented - it will be merged and the vma containing the
> > - *      area is returned, or the function will return NULL
> > - */
> > -static struct vm_area_struct
> > -*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
> > -	   struct vm_area_struct *src, unsigned long addr, unsigned long end,
> > -	   unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
> > -	   struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > -	   struct anon_vma_name *anon_name)
> > -{
> > -	struct mm_struct *mm = src->vm_mm;
> > -	struct anon_vma *anon_vma = src->anon_vma;
> > -	struct file *file = src->vm_file;
> > -	struct vm_area_struct *curr, *next, *res;
> > -	struct vm_area_struct *vma, *adjust, *remove, *remove2;
> > -	struct vm_area_struct *anon_dup = NULL;
> > -	struct vma_prepare vp;
> > -	pgoff_t vma_pgoff;
> > -	int err = 0;
> > -	bool merge_prev = false;
> > -	bool merge_next = false;
> > -	bool vma_expanded = false;
> > -	unsigned long vma_start = addr;
> > -	unsigned long vma_end = end;
> > -	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
> > -	long adj_start = 0;
> > -
> > -	/*
> > -	 * We later require that vma->vm_flags == vm_flags,
> > -	 * so this tests vma->vm_flags & VM_SPECIAL, too.
> > -	 */
> > -	if (vm_flags & VM_SPECIAL)
> > -		return NULL;
> > -
> > -	/* Does the input range span an existing VMA? (cases 5 - 8) */
> > -	curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end);
> > -
> > -	if (!curr ||			/* cases 1 - 4 */
> > -	    end == curr->vm_end)	/* cases 6 - 8, adjacent VMA */
> > -		next = vma_lookup(mm, end);
> > -	else
> > -		next = NULL;		/* case 5 */
> > -
> > -	if (prev) {
> > -		vma_start = prev->vm_start;
> > -		vma_pgoff = prev->vm_pgoff;
> > -
> > -		/* Can we merge the predecessor? */
> > -		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)
> > -		    && can_vma_merge_after(prev, vm_flags, anon_vma, file,
> > -					   pgoff, vm_userfaultfd_ctx, anon_name)) {
> > -			merge_prev = true;
> > -			vma_prev(vmi);
> > -		}
> > -	}
> > -
> > -	/* Can we merge the successor? */
> > -	if (next && mpol_equal(policy, vma_policy(next)) &&
> > -	    can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,
> > -				 vm_userfaultfd_ctx, anon_name)) {
> > -		merge_next = true;
> > -	}
> > -
> > -	/* Verify some invariant that must be enforced by the caller. */
> > -	VM_WARN_ON(prev && addr <= prev->vm_start);
> > -	VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end));
> > -	VM_WARN_ON(addr >= end);
> > -
> > -	if (!merge_prev && !merge_next)
> > -		return NULL; /* Not mergeable. */
> > -
> > -	if (merge_prev)
> > -		vma_start_write(prev);
> > -
> > -	res = vma = prev;
> > -	remove = remove2 = adjust = NULL;
> > -
> > -	/* Can we merge both the predecessor and the successor? */
> > -	if (merge_prev && merge_next &&
> > -	    is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) {
> > -		vma_start_write(next);
> > -		remove = next;				/* case 1 */
> > -		vma_end = next->vm_end;
> > -		err = dup_anon_vma(prev, next, &anon_dup);
> > -		if (curr) {				/* case 6 */
> > -			vma_start_write(curr);
> > -			remove = curr;
> > -			remove2 = next;
> > -			/*
> > -			 * Note that the dup_anon_vma below cannot overwrite err
> > -			 * since the first caller would do nothing unless next
> > -			 * has an anon_vma.
> > -			 */
> > -			if (!next->anon_vma)
> > -				err = dup_anon_vma(prev, curr, &anon_dup);
> > -		}
> > -	} else if (merge_prev) {			/* case 2 */
> > -		if (curr) {
> > -			vma_start_write(curr);
> > -			if (end == curr->vm_end) {	/* case 7 */
> > -				/*
> > -				 * can_vma_merge_after() assumed we would not be
> > -				 * removing prev vma, so it skipped the check
> > -				 * for vm_ops->close, but we are removing curr
> > -				 */
> > -				if (curr->vm_ops && curr->vm_ops->close)
> > -					err = -EINVAL;
> > -				remove = curr;
> > -			} else {			/* case 5 */
> > -				adjust = curr;
> > -				adj_start = (end - curr->vm_start);
> > -			}
> > -			if (!err)
> > -				err = dup_anon_vma(prev, curr, &anon_dup);
> > -		}
> > -	} else { /* merge_next */
> > -		vma_start_write(next);
> > -		res = next;
> > -		if (prev && addr < prev->vm_end) {	/* case 4 */
> > -			vma_start_write(prev);
> > -			vma_end = addr;
> > -			adjust = next;
> > -			adj_start = -(prev->vm_end - addr);
> > -			err = dup_anon_vma(next, prev, &anon_dup);
> > -		} else {
> > -			/*
> > -			 * Note that cases 3 and 8 are the ONLY ones where prev
> > -			 * is permitted to be (but is not necessarily) NULL.
> > -			 */
> > -			vma = next;			/* case 3 */
> > -			vma_start = addr;
> > -			vma_end = next->vm_end;
> > -			vma_pgoff = next->vm_pgoff - pglen;
> > -			if (curr) {			/* case 8 */
> > -				vma_pgoff = curr->vm_pgoff;
> > -				vma_start_write(curr);
> > -				remove = curr;
> > -				err = dup_anon_vma(next, curr, &anon_dup);
> > -			}
> > -		}
> > -	}
> > -
> > -	/* Error in anon_vma clone. */
> > -	if (err)
> > -		goto anon_vma_fail;
> > -
> > -	if (vma_start < vma->vm_start || vma_end > vma->vm_end)
> > -		vma_expanded = true;
> > -
> > -	if (vma_expanded) {
> > -		vma_iter_config(vmi, vma_start, vma_end);
> > -	} else {
> > -		vma_iter_config(vmi, adjust->vm_start + adj_start,
> > -				adjust->vm_end);
> > -	}
> > -
> > -	if (vma_iter_prealloc(vmi, vma))
> > -		goto prealloc_fail;
> > -
> > -	init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
> > -	VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
> > -		   vp.anon_vma != adjust->anon_vma);
> > -
> > -	vma_prepare(&vp);
> > -	vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start);
> > -	vma_set_range(vma, vma_start, vma_end, vma_pgoff);
> > -
> > -	if (vma_expanded)
> > -		vma_iter_store(vmi, vma);
> > -
> > -	if (adj_start) {
> > -		adjust->vm_start += adj_start;
> > -		adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
> > -		if (adj_start < 0) {
> > -			WARN_ON(vma_expanded);
> > -			vma_iter_store(vmi, next);
> > -		}
> > -	}
> > -
> > -	vma_complete(&vp, vmi, mm);
> > -	khugepaged_enter_vma(res, vm_flags);
> > -	return res;
> > -
> > -prealloc_fail:
> > -	if (anon_dup)
> > -		unlink_anon_vmas(anon_dup);
> > -
> > -anon_vma_fail:
> > -	vma_iter_set(vmi, addr);
> > -	vma_iter_load(vmi);
> > -	return NULL;
> > -}
> > -
> > -/*
> > - * Rough compatibility check to quickly see if it's even worth looking
> > - * at sharing an anon_vma.
> > - *
> > - * They need to have the same vm_file, and the flags can only differ
> > - * in things that mprotect may change.
> > - *
> > - * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
> > - * we can merge the two vma's. For example, we refuse to merge a vma if
> > - * there is a vm_ops->close() function, because that indicates that the
> > - * driver is doing some kind of reference counting. But that doesn't
> > - * really matter for the anon_vma sharing case.
> > - */
> > -static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
> > -{
> > -	return a->vm_end == b->vm_start &&
> > -		mpol_equal(vma_policy(a), vma_policy(b)) &&
> > -		a->vm_file == b->vm_file &&
> > -		!((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
> > -		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
> > -}
> > -
> > -/*
> > - * Do some basic sanity checking to see if we can re-use the anon_vma
> > - * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
> > - * the same as 'old', the other will be the new one that is trying
> > - * to share the anon_vma.
> > - *
> > - * NOTE! This runs with mmap_lock held for reading, so it is possible that
> > - * the anon_vma of 'old' is concurrently in the process of being set up
> > - * by another page fault trying to merge _that_. But that's ok: if it
> > - * is being set up, that automatically means that it will be a singleton
> > - * acceptable for merging, so we can do all of this optimistically. But
> > - * we do that READ_ONCE() to make sure that we never re-load the pointer.
> > - *
> > - * IOW: that the "list_is_singular()" test on the anon_vma_chain only
> > - * matters for the 'stable anon_vma' case (ie the thing we want to avoid
> > - * is to return an anon_vma that is "complex" due to having gone through
> > - * a fork).
> > - *
> > - * We also make sure that the two vma's are compatible (adjacent,
> > - * and with the same memory policies). That's all stable, even with just
> > - * a read lock on the mmap_lock.
> > - */
> > -static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
> > -{
> > -	if (anon_vma_compatible(a, b)) {
> > -		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
> > -
> > -		if (anon_vma && list_is_singular(&old->anon_vma_chain))
> > -			return anon_vma;
> > -	}
> > -	return NULL;
> > -}
> > +		goto out;
> >
> > -/*
> > - * find_mergeable_anon_vma is used by anon_vma_prepare, to check
> > - * neighbouring vmas for a suitable anon_vma, before it goes off
> > - * to allocate a new anon_vma.  It checks because a repetitive
> > - * sequence of mprotects and faults may otherwise lead to distinct
> > - * anon_vmas being allocated, preventing vma merge in subsequent
> > - * mprotect.
> > - */
> > -struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
> > -{
> > -	struct anon_vma *anon_vma = NULL;
> > -	struct vm_area_struct *prev, *next;
> > -	VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end);
> > -
> > -	/* Try next first. */
> > -	next = vma_iter_load(&vmi);
> > -	if (next) {
> > -		anon_vma = reusable_anon_vma(next, vma, next);
> > -		if (anon_vma)
> > -			return anon_vma;
> > -	}
> > +	mm->brk = brk;
> > +	if (mm->def_flags & VM_LOCKED)
> > +		populate = true;
> >
> > -	prev = vma_prev(&vmi);
> > -	VM_BUG_ON_VMA(prev != vma, vma);
> > -	prev = vma_prev(&vmi);
> > -	/* Try prev next. */
> > -	if (prev)
> > -		anon_vma = reusable_anon_vma(prev, prev, vma);
> > +success:
> > +	mmap_write_unlock(mm);
> > +success_unlocked:
> > +	userfaultfd_unmap_complete(mm, &uf);
> > +	if (populate)
> > +		mm_populate(oldbrk, newbrk - oldbrk);
> > +	return brk;
> >
> > -	/*
> > -	 * We might reach here with anon_vma == NULL if we can't find
> > -	 * any reusable anon_vma.
> > -	 * There's no absolute need to look only at touching neighbours:
> > -	 * we could search further afield for "compatible" anon_vmas.
> > -	 * But it would probably just be a waste of time searching,
> > -	 * or lead to too many vmas hanging off the same anon_vma.
> > -	 * We're trying to allow mprotect remerging later on,
> > -	 * not trying to minimize memory used for anon_vmas.
> > -	 */
> > -	return anon_vma;
> > +out:
> > +	mm->brk = origbrk;
> > +	mmap_write_unlock(mm);
> > +	return origbrk;
> >  }
> >
> >  /*
> > @@ -1519,85 +547,6 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
> >  }
> >  #endif /* __ARCH_WANT_SYS_OLD_MMAP */
> >
> > -static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
> > -{
> > -	return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
> > -}
> > -
> > -static bool vma_is_shared_writable(struct vm_area_struct *vma)
> > -{
> > -	return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
> > -		(VM_WRITE | VM_SHARED);
> > -}
> > -
> > -static bool vma_fs_can_writeback(struct vm_area_struct *vma)
> > -{
> > -	/* No managed pages to writeback. */
> > -	if (vma->vm_flags & VM_PFNMAP)
> > -		return false;
> > -
> > -	return vma->vm_file && vma->vm_file->f_mapping &&
> > -		mapping_can_writeback(vma->vm_file->f_mapping);
> > -}
> > -
> > -/*
> > - * Does this VMA require the underlying folios to have their dirty state
> > - * tracked?
> > - */
> > -bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
> > -{
> > -	/* Only shared, writable VMAs require dirty tracking. */
> > -	if (!vma_is_shared_writable(vma))
> > -		return false;
> > -
> > -	/* Does the filesystem need to be notified? */
> > -	if (vm_ops_needs_writenotify(vma->vm_ops))
> > -		return true;
> > -
> > -	/*
> > -	 * Even if the filesystem doesn't indicate a need for writenotify, if it
> > -	 * can writeback, dirty tracking is still required.
> > -	 */
> > -	return vma_fs_can_writeback(vma);
> > -}
> > -
> > -/*
> > - * Some shared mappings will want the pages marked read-only
> > - * to track write events. If so, we'll downgrade vm_page_prot
> > - * to the private version (using protection_map[] without the
> > - * VM_SHARED bit).
> > - */
> > -bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
> > -{
> > -	/* If it was private or non-writable, the write bit is already clear */
> > -	if (!vma_is_shared_writable(vma))
> > -		return false;
> > -
> > -	/* The backer wishes to know when pages are first written to? */
> > -	if (vm_ops_needs_writenotify(vma->vm_ops))
> > -		return true;
> > -
> > -	/* The open routine did something to the protections that pgprot_modify
> > -	 * won't preserve? */
> > -	if (pgprot_val(vm_page_prot) !=
> > -	    pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
> > -		return false;
> > -
> > -	/*
> > -	 * Do we need to track softdirty? hugetlb does not support softdirty
> > -	 * tracking yet.
> > -	 */
> > -	if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
> > -		return true;
> > -
> > -	/* Do we need write faults for uffd-wp tracking? */
> > -	if (userfaultfd_wp(vma))
> > -		return true;
> > -
> > -	/* Can the mapping track the dirty pages? */
> > -	return vma_fs_can_writeback(vma);
> > -}
> > -
> >  /*
> >   * We account for memory if it's a private writeable mapping,
> >   * not hugepages and VM_NORESERVE wasn't set.
> > @@ -2238,566 +1187,129 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address)
> >  				anon_vma_interval_tree_post_update_vma(vma);
> >  				spin_unlock(&mm->page_table_lock);
> >
> > -				perf_event_mmap(vma);
> > -			}
> > -		}
> > -	}
> > -	anon_vma_unlock_write(vma->anon_vma);
> > -	vma_iter_free(&vmi);
> > -	validate_mm(mm);
> > -	return error;
> > -}
> > -
> > -/* enforced gap between the expanding stack and other mappings. */
> > -unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
> > -
> > -static int __init cmdline_parse_stack_guard_gap(char *p)
> > -{
> > -	unsigned long val;
> > -	char *endptr;
> > -
> > -	val = simple_strtoul(p, &endptr, 10);
> > -	if (!*endptr)
> > -		stack_guard_gap = val << PAGE_SHIFT;
> > -
> > -	return 1;
> > -}
> > -__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
> > -
> > -#ifdef CONFIG_STACK_GROWSUP
> > -int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
> > -{
> > -	return expand_upwards(vma, address);
> > -}
> > -
> > -struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> > -{
> > -	struct vm_area_struct *vma, *prev;
> > -
> > -	addr &= PAGE_MASK;
> > -	vma = find_vma_prev(mm, addr, &prev);
> > -	if (vma && (vma->vm_start <= addr))
> > -		return vma;
> > -	if (!prev)
> > -		return NULL;
> > -	if (expand_stack_locked(prev, addr))
> > -		return NULL;
> > -	if (prev->vm_flags & VM_LOCKED)
> > -		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
> > -	return prev;
> > -}
> > -#else
> > -int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
> > -{
> > -	return expand_downwards(vma, address);
> > -}
> > -
> > -struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> > -{
> > -	struct vm_area_struct *vma;
> > -	unsigned long start;
> > -
> > -	addr &= PAGE_MASK;
> > -	vma = find_vma(mm, addr);
> > -	if (!vma)
> > -		return NULL;
> > -	if (vma->vm_start <= addr)
> > -		return vma;
> > -	start = vma->vm_start;
> > -	if (expand_stack_locked(vma, addr))
> > -		return NULL;
> > -	if (vma->vm_flags & VM_LOCKED)
> > -		populate_vma_page_range(vma, addr, start, NULL);
> > -	return vma;
> > -}
> > -#endif
> > -
> > -#if defined(CONFIG_STACK_GROWSUP)
> > -
> > -#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
> > -#define vma_expand_down(vma, addr) (-EFAULT)
> > -
> > -#else
> > -
> > -#define vma_expand_up(vma,addr) (-EFAULT)
> > -#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
> > -
> > -#endif
> > -
> > -/*
> > - * expand_stack(): legacy interface for page faulting. Don't use unless
> > - * you have to.
> > - *
> > - * This is called with the mm locked for reading, drops the lock, takes
> > - * the lock for writing, tries to look up a vma again, expands it if
> > - * necessary, and downgrades the lock to reading again.
> > - *
> > - * If no vma is found or it can't be expanded, it returns NULL and has
> > - * dropped the lock.
> > - */
> > -struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
> > -{
> > -	struct vm_area_struct *vma, *prev;
> > -
> > -	mmap_read_unlock(mm);
> > -	if (mmap_write_lock_killable(mm))
> > -		return NULL;
> > -
> > -	vma = find_vma_prev(mm, addr, &prev);
> > -	if (vma && vma->vm_start <= addr)
> > -		goto success;
> > -
> > -	if (prev && !vma_expand_up(prev, addr)) {
> > -		vma = prev;
> > -		goto success;
> > -	}
> > -
> > -	if (vma && !vma_expand_down(vma, addr))
> > -		goto success;
> > -
> > -	mmap_write_unlock(mm);
> > -	return NULL;
> > -
> > -success:
> > -	mmap_write_downgrade(mm);
> > -	return vma;
> > -}
> > -
> > -/*
> > - * Ok - we have the memory areas we should free on a maple tree so release them,
> > - * and do the vma updates.
> > - *
> > - * Called with the mm semaphore held.
> > - */
> > -static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
> > -{
> > -	unsigned long nr_accounted = 0;
> > -	struct vm_area_struct *vma;
> > -
> > -	/* Update high watermark before we lower total_vm */
> > -	update_hiwater_vm(mm);
> > -	mas_for_each(mas, vma, ULONG_MAX) {
> > -		long nrpages = vma_pages(vma);
> > -
> > -		if (vma->vm_flags & VM_ACCOUNT)
> > -			nr_accounted += nrpages;
> > -		vm_stat_account(mm, vma->vm_flags, -nrpages);
> > -		remove_vma(vma, false);
> > -	}
> > -	vm_unacct_memory(nr_accounted);
> > -}
> > -
> > -/*
> > - * Get rid of page table information in the indicated region.
> > - *
> > - * Called with the mm semaphore held.
> > - */
> > -static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> > -		struct vm_area_struct *vma, struct vm_area_struct *prev,
> > -		struct vm_area_struct *next, unsigned long start,
> > -		unsigned long end, unsigned long tree_end, bool mm_wr_locked)
> > -{
> > -	struct mmu_gather tlb;
> > -	unsigned long mt_start = mas->index;
> > -
> > -	lru_add_drain();
> > -	tlb_gather_mmu(&tlb, mm);
> > -	update_hiwater_rss(mm);
> > -	unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked);
> > -	mas_set(mas, mt_start);
> > -	free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
> > -				 next ? next->vm_start : USER_PGTABLES_CEILING,
> > -				 mm_wr_locked);
> > -	tlb_finish_mmu(&tlb);
> > -}
> > -
> > -/*
> > - * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
> > - * has already been checked or doesn't make sense to fail.
> > - * VMA Iterator will point to the end VMA.
> > - */
> > -static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -		       unsigned long addr, int new_below)
> > -{
> > -	struct vma_prepare vp;
> > -	struct vm_area_struct *new;
> > -	int err;
> > -
> > -	WARN_ON(vma->vm_start >= addr);
> > -	WARN_ON(vma->vm_end <= addr);
> > -
> > -	if (vma->vm_ops && vma->vm_ops->may_split) {
> > -		err = vma->vm_ops->may_split(vma, addr);
> > -		if (err)
> > -			return err;
> > -	}
> > -
> > -	new = vm_area_dup(vma);
> > -	if (!new)
> > -		return -ENOMEM;
> > -
> > -	if (new_below) {
> > -		new->vm_end = addr;
> > -	} else {
> > -		new->vm_start = addr;
> > -		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
> > -	}
> > -
> > -	err = -ENOMEM;
> > -	vma_iter_config(vmi, new->vm_start, new->vm_end);
> > -	if (vma_iter_prealloc(vmi, new))
> > -		goto out_free_vma;
> > -
> > -	err = vma_dup_policy(vma, new);
> > -	if (err)
> > -		goto out_free_vmi;
> > -
> > -	err = anon_vma_clone(new, vma);
> > -	if (err)
> > -		goto out_free_mpol;
> > -
> > -	if (new->vm_file)
> > -		get_file(new->vm_file);
> > -
> > -	if (new->vm_ops && new->vm_ops->open)
> > -		new->vm_ops->open(new);
> > -
> > -	vma_start_write(vma);
> > -	vma_start_write(new);
> > -
> > -	init_vma_prep(&vp, vma);
> > -	vp.insert = new;
> > -	vma_prepare(&vp);
> > -	vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
> > -
> > -	if (new_below) {
> > -		vma->vm_start = addr;
> > -		vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
> > -	} else {
> > -		vma->vm_end = addr;
> > -	}
> > -
> > -	/* vma_complete stores the new vma */
> > -	vma_complete(&vp, vmi, vma->vm_mm);
> > -
> > -	/* Success. */
> > -	if (new_below)
> > -		vma_next(vmi);
> > -	return 0;
> > -
> > -out_free_mpol:
> > -	mpol_put(vma_policy(new));
> > -out_free_vmi:
> > -	vma_iter_free(vmi);
> > -out_free_vma:
> > -	vm_area_free(new);
> > -	return err;
> > -}
> > -
> > -/*
> > - * Split a vma into two pieces at address 'addr', a new vma is allocated
> > - * either for the first part or the tail.
> > - */
> > -static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -		     unsigned long addr, int new_below)
> > -{
> > -	if (vma->vm_mm->map_count >= sysctl_max_map_count)
> > -		return -ENOMEM;
> > -
> > -	return __split_vma(vmi, vma, addr, new_below);
> > -}
> > -
> > -/*
> > - * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
> > - * context and anonymous VMA name within the range [start, end).
> > - *
> > - * As a result, we might be able to merge the newly modified VMA range with an
> > - * adjacent VMA with identical properties.
> > - *
> > - * If no merge is possible and the range does not span the entirety of the VMA,
> > - * we then need to split the VMA to accommodate the change.
> > - *
> > - * The function returns either the merged VMA, the original VMA if a split was
> > - * required instead, or an error if the split failed.
> > - */
> > -struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> > -				  struct vm_area_struct *prev,
> > -				  struct vm_area_struct *vma,
> > -				  unsigned long start, unsigned long end,
> > -				  unsigned long vm_flags,
> > -				  struct mempolicy *policy,
> > -				  struct vm_userfaultfd_ctx uffd_ctx,
> > -				  struct anon_vma_name *anon_name)
> > -{
> > -	pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> > -	struct vm_area_struct *merged;
> > -
> > -	merged = vma_merge(vmi, prev, vma, start, end, vm_flags,
> > -			   pgoff, policy, uffd_ctx, anon_name);
> > -	if (merged)
> > -		return merged;
> > -
> > -	if (vma->vm_start < start) {
> > -		int err = split_vma(vmi, vma, start, 1);
> > -
> > -		if (err)
> > -			return ERR_PTR(err);
> > +				perf_event_mmap(vma);
> > +			}
> > +		}
> >  	}
> > +	anon_vma_unlock_write(vma->anon_vma);
> > +	vma_iter_free(&vmi);
> > +	validate_mm(mm);
> > +	return error;
> > +}
> >
> > -	if (vma->vm_end > end) {
> > -		int err = split_vma(vmi, vma, end, 0);
> > +/* enforced gap between the expanding stack and other mappings. */
> > +unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
> >
> > -		if (err)
> > -			return ERR_PTR(err);
> > -	}
> > +static int __init cmdline_parse_stack_guard_gap(char *p)
> > +{
> > +	unsigned long val;
> > +	char *endptr;
> >
> > -	return vma;
> > +	val = simple_strtoul(p, &endptr, 10);
> > +	if (!*endptr)
> > +		stack_guard_gap = val << PAGE_SHIFT;
> > +
> > +	return 1;
> >  }
> > +__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
> >
> > -/*
> > - * Attempt to merge a newly mapped VMA with those adjacent to it. The caller
> > - * must ensure that [start, end) does not overlap any existing VMA.
> > - */
> > -static struct vm_area_struct
> > -*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
> > -		   struct vm_area_struct *vma, unsigned long start,
> > -		   unsigned long end, pgoff_t pgoff)
> > +#ifdef CONFIG_STACK_GROWSUP
> > +int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
> >  {
> > -	return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,
> > -			 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> > +	return expand_upwards(vma, address);
> >  }
> >
> > -/*
> > - * Expand vma by delta bytes, potentially merging with an immediately adjacent
> > - * VMA with identical properties.
> > - */
> > -struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> > -					struct vm_area_struct *vma,
> > -					unsigned long delta)
> > +struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> >  {
> > -	pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);
> > +	struct vm_area_struct *vma, *prev;
> >
> > -	/* vma is specified as prev, so case 1 or 2 will apply. */
> > -	return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,
> > -			 vma->vm_flags, pgoff, vma_policy(vma),
> > -			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> > +	addr &= PAGE_MASK;
> > +	vma = find_vma_prev(mm, addr, &prev);
> > +	if (vma && (vma->vm_start <= addr))
> > +		return vma;
> > +	if (!prev)
> > +		return NULL;
> > +	if (expand_stack_locked(prev, addr))
> > +		return NULL;
> > +	if (prev->vm_flags & VM_LOCKED)
> > +		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
> > +	return prev;
> >  }
> > -
> > -/*
> > - * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
> > - * @vmi: The vma iterator
> > - * @vma: The starting vm_area_struct
> > - * @mm: The mm_struct
> > - * @start: The aligned start address to munmap.
> > - * @end: The aligned end address to munmap.
> > - * @uf: The userfaultfd list_head
> > - * @unlock: Set to true to drop the mmap_lock.  unlocking only happens on
> > - * success.
> > - *
> > - * Return: 0 on success and drops the lock if so directed, error and leaves the
> > - * lock held otherwise.
> > - */
> > -static int
> > -do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > -		    struct mm_struct *mm, unsigned long start,
> > -		    unsigned long end, struct list_head *uf, bool unlock)
> > +#else
> > +int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
> >  {
> > -	struct vm_area_struct *prev, *next = NULL;
> > -	struct maple_tree mt_detach;
> > -	int count = 0;
> > -	int error = -ENOMEM;
> > -	unsigned long locked_vm = 0;
> > -	MA_STATE(mas_detach, &mt_detach, 0, 0);
> > -	mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
> > -	mt_on_stack(mt_detach);
> > -
> > -	/*
> > -	 * If we need to split any vma, do it now to save pain later.
> > -	 *
> > -	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
> > -	 * unmapped vm_area_struct will remain in use: so lower split_vma
> > -	 * places tmp vma above, and higher split_vma places tmp vma below.
> > -	 */
> > -
> > -	/* Does it split the first one? */
> > -	if (start > vma->vm_start) {
> > -
> > -		/*
> > -		 * Make sure that map_count on return from munmap() will
> > -		 * not exceed its limit; but let map_count go just above
> > -		 * its limit temporarily, to help free resources as expected.
> > -		 */
> > -		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
> > -			goto map_count_exceeded;
> > -
> > -		error = __split_vma(vmi, vma, start, 1);
> > -		if (error)
> > -			goto start_split_failed;
> > -	}
> > -
> > -	/*
> > -	 * Detach a range of VMAs from the mm. Using next as a temp variable as
> > -	 * it is always overwritten.
> > -	 */
> > -	next = vma;
> > -	do {
> > -		/* Does it split the end? */
> > -		if (next->vm_end > end) {
> > -			error = __split_vma(vmi, next, end, 0);
> > -			if (error)
> > -				goto end_split_failed;
> > -		}
> > -		vma_start_write(next);
> > -		mas_set(&mas_detach, count);
> > -		error = mas_store_gfp(&mas_detach, next, GFP_KERNEL);
> > -		if (error)
> > -			goto munmap_gather_failed;
> > -		vma_mark_detached(next, true);
> > -		if (next->vm_flags & VM_LOCKED)
> > -			locked_vm += vma_pages(next);
> > +	return expand_downwards(vma, address);
> > +}
> >
> > -		count++;
> > -		if (unlikely(uf)) {
> > -			/*
> > -			 * If userfaultfd_unmap_prep returns an error the vmas
> > -			 * will remain split, but userland will get a
> > -			 * highly unexpected error anyway. This is no
> > -			 * different than the case where the first of the two
> > -			 * __split_vma fails, but we don't undo the first
> > -			 * split, despite we could. This is unlikely enough
> > -			 * failure that it's not worth optimizing it for.
> > -			 */
> > -			error = userfaultfd_unmap_prep(next, start, end, uf);
> > +struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> > +{
> > +	struct vm_area_struct *vma;
> > +	unsigned long start;
> >
> > -			if (error)
> > -				goto userfaultfd_error;
> > -		}
> > -#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> > -		BUG_ON(next->vm_start < start);
> > -		BUG_ON(next->vm_start > end);
> > -#endif
> > -	} for_each_vma_range(*vmi, next, end);
> > -
> > -#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> > -	/* Make sure no VMAs are about to be lost. */
> > -	{
> > -		MA_STATE(test, &mt_detach, 0, 0);
> > -		struct vm_area_struct *vma_mas, *vma_test;
> > -		int test_count = 0;
> > -
> > -		vma_iter_set(vmi, start);
> > -		rcu_read_lock();
> > -		vma_test = mas_find(&test, count - 1);
> > -		for_each_vma_range(*vmi, vma_mas, end) {
> > -			BUG_ON(vma_mas != vma_test);
> > -			test_count++;
> > -			vma_test = mas_next(&test, count - 1);
> > -		}
> > -		rcu_read_unlock();
> > -		BUG_ON(count != test_count);
> > -	}
> > +	addr &= PAGE_MASK;
> > +	vma = find_vma(mm, addr);
> > +	if (!vma)
> > +		return NULL;
> > +	if (vma->vm_start <= addr)
> > +		return vma;
> > +	start = vma->vm_start;
> > +	if (expand_stack_locked(vma, addr))
> > +		return NULL;
> > +	if (vma->vm_flags & VM_LOCKED)
> > +		populate_vma_page_range(vma, addr, start, NULL);
> > +	return vma;
> > +}
> >  #endif
> >
> > -	while (vma_iter_addr(vmi) > start)
> > -		vma_iter_prev_range(vmi);
> > -
> > -	error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
> > -	if (error)
> > -		goto clear_tree_failed;
> > -
> > -	/* Point of no return */
> > -	mm->locked_vm -= locked_vm;
> > -	mm->map_count -= count;
> > -	if (unlock)
> > -		mmap_write_downgrade(mm);
> > +#if defined(CONFIG_STACK_GROWSUP)
> >
> > -	prev = vma_iter_prev_range(vmi);
> > -	next = vma_next(vmi);
> > -	if (next)
> > -		vma_iter_prev_range(vmi);
> > +#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
> > +#define vma_expand_down(vma, addr) (-EFAULT)
> >
> > -	/*
> > -	 * We can free page tables without write-locking mmap_lock because VMAs
> > -	 * were isolated before we downgraded mmap_lock.
> > -	 */
> > -	mas_set(&mas_detach, 1);
> > -	unmap_region(mm, &mas_detach, vma, prev, next, start, end, count,
> > -		     !unlock);
> > -	/* Statistics and freeing VMAs */
> > -	mas_set(&mas_detach, 0);
> > -	remove_mt(mm, &mas_detach);
> > -	validate_mm(mm);
> > -	if (unlock)
> > -		mmap_read_unlock(mm);
> > +#else
> >
> > -	__mt_destroy(&mt_detach);
> > -	return 0;
> > +#define vma_expand_up(vma,addr) (-EFAULT)
> > +#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
> >
> > -clear_tree_failed:
> > -userfaultfd_error:
> > -munmap_gather_failed:
> > -end_split_failed:
> > -	mas_set(&mas_detach, 0);
> > -	mas_for_each(&mas_detach, next, end)
> > -		vma_mark_detached(next, false);
> > -
> > -	__mt_destroy(&mt_detach);
> > -start_split_failed:
> > -map_count_exceeded:
> > -	validate_mm(mm);
> > -	return error;
> > -}
> > +#endif
> >
> >  /*
> > - * do_vmi_munmap() - munmap a given range.
> > - * @vmi: The vma iterator
> > - * @mm: The mm_struct
> > - * @start: The start address to munmap
> > - * @len: The length of the range to munmap
> > - * @uf: The userfaultfd list_head
> > - * @unlock: set to true if the user wants to drop the mmap_lock on success
> > + * expand_stack(): legacy interface for page faulting. Don't use unless
> > + * you have to.
> >   *
> > - * This function takes a @mas that is either pointing to the previous VMA or set
> > - * to MA_START and sets it up to remove the mapping(s).  The @len will be
> > - * aligned and any arch_unmap work will be preformed.
> > + * This is called with the mm locked for reading, drops the lock, takes
> > + * the lock for writing, tries to look up a vma again, expands it if
> > + * necessary, and downgrades the lock to reading again.
> >   *
> > - * Return: 0 on success and drops the lock if so directed, error and leaves the
> > - * lock held otherwise.
> > + * If no vma is found or it can't be expanded, it returns NULL and has
> > + * dropped the lock.
> >   */
> > -int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
> > -		  unsigned long start, size_t len, struct list_head *uf,
> > -		  bool unlock)
> > +struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
> >  {
> > -	unsigned long end;
> > -	struct vm_area_struct *vma;
> > +	struct vm_area_struct *vma, *prev;
> >
> > -	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
> > -		return -EINVAL;
> > +	mmap_read_unlock(mm);
> > +	if (mmap_write_lock_killable(mm))
> > +		return NULL;
> >
> > -	end = start + PAGE_ALIGN(len);
> > -	if (end == start)
> > -		return -EINVAL;
> > +	vma = find_vma_prev(mm, addr, &prev);
> > +	if (vma && vma->vm_start <= addr)
> > +		goto success;
> >
> > -	/*
> > -	 * Check if memory is sealed before arch_unmap.
> > -	 * Prevent unmapping a sealed VMA.
> > -	 * can_modify_mm assumes we have acquired the lock on MM.
> > -	 */
> > -	if (unlikely(!can_modify_mm(mm, start, end)))
> > -		return -EPERM;
> > +	if (prev && !vma_expand_up(prev, addr)) {
> > +		vma = prev;
> > +		goto success;
> > +	}
> >
> > -	 /* arch_unmap() might do unmaps itself.  */
> > -	arch_unmap(mm, start, end);
> > +	if (vma && !vma_expand_down(vma, addr))
> > +		goto success;
> >
> > -	/* Find the first overlapping VMA */
> > -	vma = vma_find(vmi, end);
> > -	if (!vma) {
> > -		if (unlock)
> > -			mmap_write_unlock(mm);
> > -		return 0;
> > -	}
> > +	mmap_write_unlock(mm);
> > +	return NULL;
> >
> > -	return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
> > +success:
> > +	mmap_write_downgrade(mm);
> > +	return vma;
> >  }
> >
> >  /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
> > @@ -3460,92 +1972,6 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
> >  	return 0;
> >  }
> >
> > -/*
> > - * Copy the vma structure to a new location in the same mm,
> > - * prior to moving page table entries, to effect an mremap move.
> > - */
> > -struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> > -	unsigned long addr, unsigned long len, pgoff_t pgoff,
> > -	bool *need_rmap_locks)
> > -{
> > -	struct vm_area_struct *vma = *vmap;
> > -	unsigned long vma_start = vma->vm_start;
> > -	struct mm_struct *mm = vma->vm_mm;
> > -	struct vm_area_struct *new_vma, *prev;
> > -	bool faulted_in_anon_vma = true;
> > -	VMA_ITERATOR(vmi, mm, addr);
> > -
> > -	/*
> > -	 * If anonymous vma has not yet been faulted, update new pgoff
> > -	 * to match new location, to increase its chance of merging.
> > -	 */
> > -	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
> > -		pgoff = addr >> PAGE_SHIFT;
> > -		faulted_in_anon_vma = false;
> > -	}
> > -
> > -	new_vma = find_vma_prev(mm, addr, &prev);
> > -	if (new_vma && new_vma->vm_start < addr + len)
> > -		return NULL;	/* should never get here */
> > -
> > -	new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff);
> > -	if (new_vma) {
> > -		/*
> > -		 * Source vma may have been merged into new_vma
> > -		 */
> > -		if (unlikely(vma_start >= new_vma->vm_start &&
> > -			     vma_start < new_vma->vm_end)) {
> > -			/*
> > -			 * The only way we can get a vma_merge with
> > -			 * self during an mremap is if the vma hasn't
> > -			 * been faulted in yet and we were allowed to
> > -			 * reset the dst vma->vm_pgoff to the
> > -			 * destination address of the mremap to allow
> > -			 * the merge to happen. mremap must change the
> > -			 * vm_pgoff linearity between src and dst vmas
> > -			 * (in turn preventing a vma_merge) to be
> > -			 * safe. It is only safe to keep the vm_pgoff
> > -			 * linear if there are no pages mapped yet.
> > -			 */
> > -			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
> > -			*vmap = vma = new_vma;
> > -		}
> > -		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
> > -	} else {
> > -		new_vma = vm_area_dup(vma);
> > -		if (!new_vma)
> > -			goto out;
> > -		vma_set_range(new_vma, addr, addr + len, pgoff);
> > -		if (vma_dup_policy(vma, new_vma))
> > -			goto out_free_vma;
> > -		if (anon_vma_clone(new_vma, vma))
> > -			goto out_free_mempol;
> > -		if (new_vma->vm_file)
> > -			get_file(new_vma->vm_file);
> > -		if (new_vma->vm_ops && new_vma->vm_ops->open)
> > -			new_vma->vm_ops->open(new_vma);
> > -		if (vma_link(mm, new_vma))
> > -			goto out_vma_link;
> > -		*need_rmap_locks = false;
> > -	}
> > -	return new_vma;
> > -
> > -out_vma_link:
> > -	if (new_vma->vm_ops && new_vma->vm_ops->close)
> > -		new_vma->vm_ops->close(new_vma);
> > -
> > -	if (new_vma->vm_file)
> > -		fput(new_vma->vm_file);
> > -
> > -	unlink_anon_vmas(new_vma);
> > -out_free_mempol:
> > -	mpol_put(vma_policy(new_vma));
> > -out_free_vma:
> > -	vm_area_free(new_vma);
> > -out:
> > -	return NULL;
> > -}
> > -
> >  /*
> >   * Return true if the calling process may expand its vm space by the passed
> >   * number of pages
> > @@ -3743,203 +2169,6 @@ int install_special_mapping(struct mm_struct *mm,
> >  	return PTR_ERR_OR_ZERO(vma);
> >  }
> >
> > -static DEFINE_MUTEX(mm_all_locks_mutex);
> > -
> > -static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
> > -{
> > -	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> > -		/*
> > -		 * The LSB of head.next can't change from under us
> > -		 * because we hold the mm_all_locks_mutex.
> > -		 */
> > -		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
> > -		/*
> > -		 * We can safely modify head.next after taking the
> > -		 * anon_vma->root->rwsem. If some other vma in this mm shares
> > -		 * the same anon_vma we won't take it again.
> > -		 *
> > -		 * No need of atomic instructions here, head.next
> > -		 * can't change from under us thanks to the
> > -		 * anon_vma->root->rwsem.
> > -		 */
> > -		if (__test_and_set_bit(0, (unsigned long *)
> > -				       &anon_vma->root->rb_root.rb_root.rb_node))
> > -			BUG();
> > -	}
> > -}
> > -
> > -static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
> > -{
> > -	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> > -		/*
> > -		 * AS_MM_ALL_LOCKS can't change from under us because
> > -		 * we hold the mm_all_locks_mutex.
> > -		 *
> > -		 * Operations on ->flags have to be atomic because
> > -		 * even if AS_MM_ALL_LOCKS is stable thanks to the
> > -		 * mm_all_locks_mutex, there may be other cpus
> > -		 * changing other bitflags in parallel to us.
> > -		 */
> > -		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
> > -			BUG();
> > -		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
> > -	}
> > -}
> > -
> > -/*
> > - * This operation locks against the VM for all pte/vma/mm related
> > - * operations that could ever happen on a certain mm. This includes
> > - * vmtruncate, try_to_unmap, and all page faults.
> > - *
> > - * The caller must take the mmap_lock in write mode before calling
> > - * mm_take_all_locks(). The caller isn't allowed to release the
> > - * mmap_lock until mm_drop_all_locks() returns.
> > - *
> > - * mmap_lock in write mode is required in order to block all operations
> > - * that could modify pagetables and free pages without need of
> > - * altering the vma layout. It's also needed in write mode to avoid new
> > - * anon_vmas to be associated with existing vmas.
> > - *
> > - * A single task can't take more than one mm_take_all_locks() in a row
> > - * or it would deadlock.
> > - *
> > - * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
> > - * mapping->flags avoid to take the same lock twice, if more than one
> > - * vma in this mm is backed by the same anon_vma or address_space.
> > - *
> > - * We take locks in following order, accordingly to comment at beginning
> > - * of mm/rmap.c:
> > - *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
> > - *     hugetlb mapping);
> > - *   - all vmas marked locked
> > - *   - all i_mmap_rwsem locks;
> > - *   - all anon_vma->rwseml
> > - *
> > - * We can take all locks within these types randomly because the VM code
> > - * doesn't nest them and we protected from parallel mm_take_all_locks() by
> > - * mm_all_locks_mutex.
> > - *
> > - * mm_take_all_locks() and mm_drop_all_locks are expensive operations
> > - * that may have to take thousand of locks.
> > - *
> > - * mm_take_all_locks() can fail if it's interrupted by signals.
> > - */
> > -int mm_take_all_locks(struct mm_struct *mm)
> > -{
> > -	struct vm_area_struct *vma;
> > -	struct anon_vma_chain *avc;
> > -	VMA_ITERATOR(vmi, mm, 0);
> > -
> > -	mmap_assert_write_locked(mm);
> > -
> > -	mutex_lock(&mm_all_locks_mutex);
> > -
> > -	/*
> > -	 * vma_start_write() does not have a complement in mm_drop_all_locks()
> > -	 * because vma_start_write() is always asymmetrical; it marks a VMA as
> > -	 * being written to until mmap_write_unlock() or mmap_write_downgrade()
> > -	 * is reached.
> > -	 */
> > -	for_each_vma(vmi, vma) {
> > -		if (signal_pending(current))
> > -			goto out_unlock;
> > -		vma_start_write(vma);
> > -	}
> > -
> > -	vma_iter_init(&vmi, mm, 0);
> > -	for_each_vma(vmi, vma) {
> > -		if (signal_pending(current))
> > -			goto out_unlock;
> > -		if (vma->vm_file && vma->vm_file->f_mapping &&
> > -				is_vm_hugetlb_page(vma))
> > -			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> > -	}
> > -
> > -	vma_iter_init(&vmi, mm, 0);
> > -	for_each_vma(vmi, vma) {
> > -		if (signal_pending(current))
> > -			goto out_unlock;
> > -		if (vma->vm_file && vma->vm_file->f_mapping &&
> > -				!is_vm_hugetlb_page(vma))
> > -			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> > -	}
> > -
> > -	vma_iter_init(&vmi, mm, 0);
> > -	for_each_vma(vmi, vma) {
> > -		if (signal_pending(current))
> > -			goto out_unlock;
> > -		if (vma->anon_vma)
> > -			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > -				vm_lock_anon_vma(mm, avc->anon_vma);
> > -	}
> > -
> > -	return 0;
> > -
> > -out_unlock:
> > -	mm_drop_all_locks(mm);
> > -	return -EINTR;
> > -}
> > -
> > -static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
> > -{
> > -	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> > -		/*
> > -		 * The LSB of head.next can't change to 0 from under
> > -		 * us because we hold the mm_all_locks_mutex.
> > -		 *
> > -		 * We must however clear the bitflag before unlocking
> > -		 * the vma so the users using the anon_vma->rb_root will
> > -		 * never see our bitflag.
> > -		 *
> > -		 * No need of atomic instructions here, head.next
> > -		 * can't change from under us until we release the
> > -		 * anon_vma->root->rwsem.
> > -		 */
> > -		if (!__test_and_clear_bit(0, (unsigned long *)
> > -					  &anon_vma->root->rb_root.rb_root.rb_node))
> > -			BUG();
> > -		anon_vma_unlock_write(anon_vma);
> > -	}
> > -}
> > -
> > -static void vm_unlock_mapping(struct address_space *mapping)
> > -{
> > -	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> > -		/*
> > -		 * AS_MM_ALL_LOCKS can't change to 0 from under us
> > -		 * because we hold the mm_all_locks_mutex.
> > -		 */
> > -		i_mmap_unlock_write(mapping);
> > -		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
> > -					&mapping->flags))
> > -			BUG();
> > -	}
> > -}
> > -
> > -/*
> > - * The mmap_lock cannot be released by the caller until
> > - * mm_drop_all_locks() returns.
> > - */
> > -void mm_drop_all_locks(struct mm_struct *mm)
> > -{
> > -	struct vm_area_struct *vma;
> > -	struct anon_vma_chain *avc;
> > -	VMA_ITERATOR(vmi, mm, 0);
> > -
> > -	mmap_assert_write_locked(mm);
> > -	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
> > -
> > -	for_each_vma(vmi, vma) {
> > -		if (vma->anon_vma)
> > -			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > -				vm_unlock_anon_vma(avc->anon_vma);
> > -		if (vma->vm_file && vma->vm_file->f_mapping)
> > -			vm_unlock_mapping(vma->vm_file->f_mapping);
> > -	}
> > -
> > -	mutex_unlock(&mm_all_locks_mutex);
> > -}
> > -
> >  /*
> >   * vma_expand_bottom() - Expands the bottom of a VMA downwards. An error will
> >   *                       arise if there is another VMA in the expanded range, or
> > diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
> > index 8982e6139d07..fc18fe274505 100644
> > --- a/mm/mmu_notifier.c
> > +++ b/mm/mmu_notifier.c
> > @@ -19,6 +19,8 @@
> >  #include <linux/sched/mm.h>
> >  #include <linux/slab.h>
> >
> > +#include "vma.h"
> > +
> >  /* global SRCU for all MMs */
> >  DEFINE_STATIC_SRCU(srcu);
> >
> > diff --git a/mm/mprotect.c b/mm/mprotect.c
> > index 222ab434da54..77951e2d0863 100644
> > --- a/mm/mprotect.c
> > +++ b/mm/mprotect.c
> > @@ -39,6 +39,7 @@
> >  #include <asm/tlb.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
> >  			     pte_t pte)
> > diff --git a/mm/mremap.c b/mm/mremap.c
> > index e7ae140fc640..09ef3eb31fbf 100644
> > --- a/mm/mremap.c
> > +++ b/mm/mremap.c
> > @@ -31,6 +31,7 @@
> >  #include <asm/pgalloc.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
> >  {
> > diff --git a/mm/mseal.c b/mm/mseal.c
> > index bf783bba8ed0..7bcceda42a1a 100644
> > --- a/mm/mseal.c
> > +++ b/mm/mseal.c
> > @@ -14,7 +14,9 @@
> >  #include <linux/mmu_context.h>
> >  #include <linux/syscalls.h>
> >  #include <linux/sched.h>
> > +
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  static inline bool vma_is_sealed(struct vm_area_struct *vma)
> >  {
> > diff --git a/mm/rmap.c b/mm/rmap.c
> > index 8616308610b9..4dec7ab3638c 100644
> > --- a/mm/rmap.c
> > +++ b/mm/rmap.c
> > @@ -83,6 +83,7 @@
> >  #include <trace/events/migrate.h>
> >
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  static struct kmem_cache *anon_vma_cachep;
> >  static struct kmem_cache *anon_vma_chain_cachep;
> > diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
> > index 950fe6b2f0f7..30be083788be 100644
> > --- a/mm/userfaultfd.c
> > +++ b/mm/userfaultfd.c
> > @@ -17,7 +17,9 @@
> >  #include <linux/shmem_fs.h>
> >  #include <asm/tlbflush.h>
> >  #include <asm/tlb.h>
> > +
> >  #include "internal.h"
> > +#include "vma.h"
> >
> >  static __always_inline
> >  bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end)
> > diff --git a/mm/vma.c b/mm/vma.c
> > new file mode 100644
> > index 000000000000..bf0546fe6eab
> > --- /dev/null
> > +++ b/mm/vma.c
> > @@ -0,0 +1,1766 @@
> > +// SPDX-License-Identifier: GPL-2.0-or-later
> > +
> > +/*
> > + * VMA-specific functions.
> > + */
> > +
> > +#include "vma_internal.h"
> > +#include "vma.h"
> > +
> > +/*
> > + * If the vma has a ->close operation then the driver probably needs to release
> > + * per-vma resources, so we don't attempt to merge those if the caller indicates
> > + * the current vma may be removed as part of the merge.
> > + */
> > +static inline bool is_mergeable_vma(struct vm_area_struct *vma,
> > +		struct file *file, unsigned long vm_flags,
> > +		struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > +		struct anon_vma_name *anon_name, bool may_remove_vma)
> > +{
> > +	/*
> > +	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
> > +	 * match the flags but dirty bit -- the caller should mark
> > +	 * merged VMA as dirty. If dirty bit won't be excluded from
> > +	 * comparison, we increase pressure on the memory system forcing
> > +	 * the kernel to generate new VMAs when old one could be
> > +	 * extended instead.
> > +	 */
> > +	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
> > +		return false;
> > +	if (vma->vm_file != file)
> > +		return false;
> > +	if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
> > +		return false;
> > +	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
> > +		return false;
> > +	if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
> > +		return false;
> > +	return true;
> > +}
> > +
> > +static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
> > +		 struct anon_vma *anon_vma2, struct vm_area_struct *vma)
> > +{
> > +	/*
> > +	 * The list_is_singular() test is to avoid merging VMA cloned from
> > +	 * parents. This can improve scalability caused by anon_vma lock.
> > +	 */
> > +	if ((!anon_vma1 || !anon_vma2) && (!vma ||
> > +		list_is_singular(&vma->anon_vma_chain)))
> > +		return true;
> > +	return anon_vma1 == anon_vma2;
> > +}
> > +
> > +/*
> > + * init_multi_vma_prep() - Initializer for struct vma_prepare
> > + * @vp: The vma_prepare struct
> > + * @vma: The vma that will be altered once locked
> > + * @next: The next vma if it is to be adjusted
> > + * @remove: The first vma to be removed
> > + * @remove2: The second vma to be removed
> > + */
> > +static void init_multi_vma_prep(struct vma_prepare *vp,
> > +				struct vm_area_struct *vma,
> > +				struct vm_area_struct *next,
> > +				struct vm_area_struct *remove,
> > +				struct vm_area_struct *remove2)
> > +{
> > +	memset(vp, 0, sizeof(struct vma_prepare));
> > +	vp->vma = vma;
> > +	vp->anon_vma = vma->anon_vma;
> > +	vp->remove = remove;
> > +	vp->remove2 = remove2;
> > +	vp->adj_next = next;
> > +	if (!vp->anon_vma && next)
> > +		vp->anon_vma = next->anon_vma;
> > +
> > +	vp->file = vma->vm_file;
> > +	if (vp->file)
> > +		vp->mapping = vma->vm_file->f_mapping;
> > +
> > +}
> > +
> > +/*
> > + * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> > + * in front of (at a lower virtual address and file offset than) the vma.
> > + *
> > + * We cannot merge two vmas if they have differently assigned (non-NULL)
> > + * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> > + *
> > + * We don't check here for the merged mmap wrapping around the end of pagecache
> > + * indices (16TB on ia32) because do_mmap() does not permit mmap's which
> > + * wrap, nor mmaps which cover the final page at index -1UL.
> > + *
> > + * We assume the vma may be removed as part of the merge.
> > + */
> > +bool
> > +can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
> > +		struct anon_vma *anon_vma, struct file *file,
> > +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > +		struct anon_vma_name *anon_name)
> > +{
> > +	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&
> > +	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> > +		if (vma->vm_pgoff == vm_pgoff)
> > +			return true;
> > +	}
> > +	return false;
> > +}
> > +
> > +/*
> > + * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> > + * beyond (at a higher virtual address and file offset than) the vma.
> > + *
> > + * We cannot merge two vmas if they have differently assigned (non-NULL)
> > + * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> > + *
> > + * We assume that vma is not removed as part of the merge.
> > + */
> > +bool
> > +can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
> > +		struct anon_vma *anon_vma, struct file *file,
> > +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > +		struct anon_vma_name *anon_name)
> > +{
> > +	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&
> > +	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> > +		pgoff_t vm_pglen;
> > +
> > +		vm_pglen = vma_pages(vma);
> > +		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
> > +			return true;
> > +	}
> > +	return false;
> > +}
> > +
> > +/*
> > + * Close a vm structure and free it.
> > + */
> > +void remove_vma(struct vm_area_struct *vma, bool unreachable)
> > +{
> > +	might_sleep();
> > +	if (vma->vm_ops && vma->vm_ops->close)
> > +		vma->vm_ops->close(vma);
> > +	if (vma->vm_file)
> > +		fput(vma->vm_file);
> > +	mpol_put(vma_policy(vma));
> > +	if (unreachable)
> > +		__vm_area_free(vma);
> > +	else
> > +		vm_area_free(vma);
> > +}
> > +
> > +/*
> > + * Get rid of page table information in the indicated region.
> > + *
> > + * Called with the mm semaphore held.
> > + */
> > +void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> > +		struct vm_area_struct *vma, struct vm_area_struct *prev,
> > +		struct vm_area_struct *next, unsigned long start,
> > +		unsigned long end, unsigned long tree_end, bool mm_wr_locked)
> > +{
> > +	struct mmu_gather tlb;
> > +	unsigned long mt_start = mas->index;
> > +
> > +	lru_add_drain();
> > +	tlb_gather_mmu(&tlb, mm);
> > +	update_hiwater_rss(mm);
> > +	unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked);
> > +	mas_set(mas, mt_start);
> > +	free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
> > +				 next ? next->vm_start : USER_PGTABLES_CEILING,
> > +				 mm_wr_locked);
> > +	tlb_finish_mmu(&tlb);
> > +}
> > +
> > +/*
> > + * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
> > + * has already been checked or doesn't make sense to fail.
> > + * VMA Iterator will point to the end VMA.
> > + */
> > +static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +		       unsigned long addr, int new_below)
> > +{
> > +	struct vma_prepare vp;
> > +	struct vm_area_struct *new;
> > +	int err;
> > +
> > +	WARN_ON(vma->vm_start >= addr);
> > +	WARN_ON(vma->vm_end <= addr);
> > +
> > +	if (vma->vm_ops && vma->vm_ops->may_split) {
> > +		err = vma->vm_ops->may_split(vma, addr);
> > +		if (err)
> > +			return err;
> > +	}
> > +
> > +	new = vm_area_dup(vma);
> > +	if (!new)
> > +		return -ENOMEM;
> > +
> > +	if (new_below) {
> > +		new->vm_end = addr;
> > +	} else {
> > +		new->vm_start = addr;
> > +		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
> > +	}
> > +
> > +	err = -ENOMEM;
> > +	vma_iter_config(vmi, new->vm_start, new->vm_end);
> > +	if (vma_iter_prealloc(vmi, new))
> > +		goto out_free_vma;
> > +
> > +	err = vma_dup_policy(vma, new);
> > +	if (err)
> > +		goto out_free_vmi;
> > +
> > +	err = anon_vma_clone(new, vma);
> > +	if (err)
> > +		goto out_free_mpol;
> > +
> > +	if (new->vm_file)
> > +		get_file(new->vm_file);
> > +
> > +	if (new->vm_ops && new->vm_ops->open)
> > +		new->vm_ops->open(new);
> > +
> > +	vma_start_write(vma);
> > +	vma_start_write(new);
> > +
> > +	init_vma_prep(&vp, vma);
> > +	vp.insert = new;
> > +	vma_prepare(&vp);
> > +	vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
> > +
> > +	if (new_below) {
> > +		vma->vm_start = addr;
> > +		vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
> > +	} else {
> > +		vma->vm_end = addr;
> > +	}
> > +
> > +	/* vma_complete stores the new vma */
> > +	vma_complete(&vp, vmi, vma->vm_mm);
> > +
> > +	/* Success. */
> > +	if (new_below)
> > +		vma_next(vmi);
> > +	return 0;
> > +
> > +out_free_mpol:
> > +	mpol_put(vma_policy(new));
> > +out_free_vmi:
> > +	vma_iter_free(vmi);
> > +out_free_vma:
> > +	vm_area_free(new);
> > +	return err;
> > +}
> > +
> > +/*
> > + * Split a vma into two pieces at address 'addr', a new vma is allocated
> > + * either for the first part or the tail.
> > + */
> > +static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +		     unsigned long addr, int new_below)
> > +{
> > +	if (vma->vm_mm->map_count >= sysctl_max_map_count)
> > +		return -ENOMEM;
> > +
> > +	return __split_vma(vmi, vma, addr, new_below);
> > +}
> > +
> > +/*
> > + * Ok - we have the memory areas we should free on a maple tree so release them,
> > + * and do the vma updates.
> > + *
> > + * Called with the mm semaphore held.
> > + */
> > +static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
> > +{
> > +	unsigned long nr_accounted = 0;
> > +	struct vm_area_struct *vma;
> > +
> > +	/* Update high watermark before we lower total_vm */
> > +	update_hiwater_vm(mm);
> > +	mas_for_each(mas, vma, ULONG_MAX) {
> > +		long nrpages = vma_pages(vma);
> > +
> > +		if (vma->vm_flags & VM_ACCOUNT)
> > +			nr_accounted += nrpages;
> > +		vm_stat_account(mm, vma->vm_flags, -nrpages);
> > +		remove_vma(vma, false);
> > +	}
> > +	vm_unacct_memory(nr_accounted);
> > +}
> > +
> > +/*
> > + * init_vma_prep() - Initializer wrapper for vma_prepare struct
> > + * @vp: The vma_prepare struct
> > + * @vma: The vma that will be altered once locked
> > + */
> > +void init_vma_prep(struct vma_prepare *vp,
> > +		   struct vm_area_struct *vma)
> > +{
> > +	init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
> > +}
> > +
> > +/*
> > + * Requires inode->i_mapping->i_mmap_rwsem
> > + */
> > +static void __remove_shared_vm_struct(struct vm_area_struct *vma,
> > +				      struct address_space *mapping)
> > +{
> > +	if (vma_is_shared_maywrite(vma))
> > +		mapping_unmap_writable(mapping);
> > +
> > +	flush_dcache_mmap_lock(mapping);
> > +	vma_interval_tree_remove(vma, &mapping->i_mmap);
> > +	flush_dcache_mmap_unlock(mapping);
> > +}
> > +
> > +/*
> > + * vma has some anon_vma assigned, and is already inserted on that
> > + * anon_vma's interval trees.
> > + *
> > + * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
> > + * vma must be removed from the anon_vma's interval trees using
> > + * anon_vma_interval_tree_pre_update_vma().
> > + *
> > + * After the update, the vma will be reinserted using
> > + * anon_vma_interval_tree_post_update_vma().
> > + *
> > + * The entire update must be protected by exclusive mmap_lock and by
> > + * the root anon_vma's mutex.
> > + */
> > +void
> > +anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
> > +{
> > +	struct anon_vma_chain *avc;
> > +
> > +	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > +		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
> > +}
> > +
> > +void
> > +anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
> > +{
> > +	struct anon_vma_chain *avc;
> > +
> > +	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > +		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
> > +}
> > +
> > +static void __vma_link_file(struct vm_area_struct *vma,
> > +			    struct address_space *mapping)
> > +{
> > +	if (vma_is_shared_maywrite(vma))
> > +		mapping_allow_writable(mapping);
> > +
> > +	flush_dcache_mmap_lock(mapping);
> > +	vma_interval_tree_insert(vma, &mapping->i_mmap);
> > +	flush_dcache_mmap_unlock(mapping);
> > +}
> > +
> > +/*
> > + * vma_prepare() - Helper function for handling locking VMAs prior to altering
> > + * @vp: The initialized vma_prepare struct
> > + */
> > +void vma_prepare(struct vma_prepare *vp)
> > +{
> > +	if (vp->file) {
> > +		uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
> > +
> > +		if (vp->adj_next)
> > +			uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
> > +				      vp->adj_next->vm_end);
> > +
> > +		i_mmap_lock_write(vp->mapping);
> > +		if (vp->insert && vp->insert->vm_file) {
> > +			/*
> > +			 * Put into interval tree now, so instantiated pages
> > +			 * are visible to arm/parisc __flush_dcache_page
> > +			 * throughout; but we cannot insert into address
> > +			 * space until vma start or end is updated.
> > +			 */
> > +			__vma_link_file(vp->insert,
> > +					vp->insert->vm_file->f_mapping);
> > +		}
> > +	}
> > +
> > +	if (vp->anon_vma) {
> > +		anon_vma_lock_write(vp->anon_vma);
> > +		anon_vma_interval_tree_pre_update_vma(vp->vma);
> > +		if (vp->adj_next)
> > +			anon_vma_interval_tree_pre_update_vma(vp->adj_next);
> > +	}
> > +
> > +	if (vp->file) {
> > +		flush_dcache_mmap_lock(vp->mapping);
> > +		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
> > +		if (vp->adj_next)
> > +			vma_interval_tree_remove(vp->adj_next,
> > +						 &vp->mapping->i_mmap);
> > +	}
> > +
> > +}
> > +
> > +/*
> > + * dup_anon_vma() - Helper function to duplicate anon_vma
> > + * @dst: The destination VMA
> > + * @src: The source VMA
> > + * @dup: Pointer to the destination VMA when successful.
> > + *
> > + * Returns: 0 on success.
> > + */
> > +static int dup_anon_vma(struct vm_area_struct *dst,
> > +			struct vm_area_struct *src, struct vm_area_struct **dup)
> > +{
> > +	/*
> > +	 * Easily overlooked: when mprotect shifts the boundary, make sure the
> > +	 * expanding vma has anon_vma set if the shrinking vma had, to cover any
> > +	 * anon pages imported.
> > +	 */
> > +	if (src->anon_vma && !dst->anon_vma) {
> > +		int ret;
> > +
> > +		vma_assert_write_locked(dst);
> > +		dst->anon_vma = src->anon_vma;
> > +		ret = anon_vma_clone(dst, src);
> > +		if (ret)
> > +			return ret;
> > +
> > +		*dup = dst;
> > +	}
> > +
> > +	return 0;
> > +}
> > +
> > +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> > +void validate_mm(struct mm_struct *mm)
> > +{
> > +	int bug = 0;
> > +	int i = 0;
> > +	struct vm_area_struct *vma;
> > +	VMA_ITERATOR(vmi, mm, 0);
> > +
> > +	mt_validate(&mm->mm_mt);
> > +	for_each_vma(vmi, vma) {
> > +#ifdef CONFIG_DEBUG_VM_RB
> > +		struct anon_vma *anon_vma = vma->anon_vma;
> > +		struct anon_vma_chain *avc;
> > +#endif
> > +		unsigned long vmi_start, vmi_end;
> > +		bool warn = 0;
> > +
> > +		vmi_start = vma_iter_addr(&vmi);
> > +		vmi_end = vma_iter_end(&vmi);
> > +		if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
> > +			warn = 1;
> > +
> > +		if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
> > +			warn = 1;
> > +
> > +		if (warn) {
> > +			pr_emerg("issue in %s\n", current->comm);
> > +			dump_stack();
> > +			dump_vma(vma);
> > +			pr_emerg("tree range: %px start %lx end %lx\n", vma,
> > +				 vmi_start, vmi_end - 1);
> > +			vma_iter_dump_tree(&vmi);
> > +		}
> > +
> > +#ifdef CONFIG_DEBUG_VM_RB
> > +		if (anon_vma) {
> > +			anon_vma_lock_read(anon_vma);
> > +			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > +				anon_vma_interval_tree_verify(avc);
> > +			anon_vma_unlock_read(anon_vma);
> > +		}
> > +#endif
> > +		i++;
> > +	}
> > +	if (i != mm->map_count) {
> > +		pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
> > +		bug = 1;
> > +	}
> > +	VM_BUG_ON_MM(bug, mm);
> > +}
> > +#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
> > +
> > +/*
> > + * vma_expand - Expand an existing VMA
> > + *
> > + * @vmi: The vma iterator
> > + * @vma: The vma to expand
> > + * @start: The start of the vma
> > + * @end: The exclusive end of the vma
> > + * @pgoff: The page offset of vma
> > + * @next: The current of next vma.
> > + *
> > + * Expand @vma to @start and @end.  Can expand off the start and end.  Will
> > + * expand over @next if it's different from @vma and @end == @next->vm_end.
> > + * Checking if the @vma can expand and merge with @next needs to be handled by
> > + * the caller.
> > + *
> > + * Returns: 0 on success
> > + */
> > +int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +	       unsigned long start, unsigned long end, pgoff_t pgoff,
> > +	       struct vm_area_struct *next)
> > +{
> > +	struct vm_area_struct *anon_dup = NULL;
> > +	bool remove_next = false;
> > +	struct vma_prepare vp;
> > +
> > +	vma_start_write(vma);
> > +	if (next && (vma != next) && (end == next->vm_end)) {
> > +		int ret;
> > +
> > +		remove_next = true;
> > +		vma_start_write(next);
> > +		ret = dup_anon_vma(vma, next, &anon_dup);
> > +		if (ret)
> > +			return ret;
> > +	}
> > +
> > +	init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL);
> > +	/* Not merging but overwriting any part of next is not handled. */
> > +	VM_WARN_ON(next && !vp.remove &&
> > +		  next != vma && end > next->vm_start);
> > +	/* Only handles expanding */
> > +	VM_WARN_ON(vma->vm_start < start || vma->vm_end > end);
> > +
> > +	/* Note: vma iterator must be pointing to 'start' */
> > +	vma_iter_config(vmi, start, end);
> > +	if (vma_iter_prealloc(vmi, vma))
> > +		goto nomem;
> > +
> > +	vma_prepare(&vp);
> > +	vma_adjust_trans_huge(vma, start, end, 0);
> > +	vma_set_range(vma, start, end, pgoff);
> > +	vma_iter_store(vmi, vma);
> > +
> > +	vma_complete(&vp, vmi, vma->vm_mm);
> > +	return 0;
> > +
> > +nomem:
> > +	if (anon_dup)
> > +		unlink_anon_vmas(anon_dup);
> > +	return -ENOMEM;
> > +}
> > +
> > +/*
> > + * vma_shrink() - Reduce an existing VMAs memory area
> > + * @vmi: The vma iterator
> > + * @vma: The VMA to modify
> > + * @start: The new start
> > + * @end: The new end
> > + *
> > + * Returns: 0 on success, -ENOMEM otherwise
> > + */
> > +int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +	       unsigned long start, unsigned long end, pgoff_t pgoff)
> > +{
> > +	struct vma_prepare vp;
> > +
> > +	WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
> > +
> > +	if (vma->vm_start < start)
> > +		vma_iter_config(vmi, vma->vm_start, start);
> > +	else
> > +		vma_iter_config(vmi, end, vma->vm_end);
> > +
> > +	if (vma_iter_prealloc(vmi, NULL))
> > +		return -ENOMEM;
> > +
> > +	vma_start_write(vma);
> > +
> > +	init_vma_prep(&vp, vma);
> > +	vma_prepare(&vp);
> > +	vma_adjust_trans_huge(vma, start, end, 0);
> > +
> > +	vma_iter_clear(vmi);
> > +	vma_set_range(vma, start, end, pgoff);
> > +	vma_complete(&vp, vmi, vma->vm_mm);
> > +	return 0;
> > +}
> > +
> > +/*
> > + * vma_complete- Helper function for handling the unlocking after altering VMAs,
> > + * or for inserting a VMA.
> > + *
> > + * @vp: The vma_prepare struct
> > + * @vmi: The vma iterator
> > + * @mm: The mm_struct
> > + */
> > +void vma_complete(struct vma_prepare *vp,
> > +		  struct vma_iterator *vmi, struct mm_struct *mm)
> > +{
> > +	if (vp->file) {
> > +		if (vp->adj_next)
> > +			vma_interval_tree_insert(vp->adj_next,
> > +						 &vp->mapping->i_mmap);
> > +		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
> > +		flush_dcache_mmap_unlock(vp->mapping);
> > +	}
> > +
> > +	if (vp->remove && vp->file) {
> > +		__remove_shared_vm_struct(vp->remove, vp->mapping);
> > +		if (vp->remove2)
> > +			__remove_shared_vm_struct(vp->remove2, vp->mapping);
> > +	} else if (vp->insert) {
> > +		/*
> > +		 * split_vma has split insert from vma, and needs
> > +		 * us to insert it before dropping the locks
> > +		 * (it may either follow vma or precede it).
> > +		 */
> > +		vma_iter_store(vmi, vp->insert);
> > +		mm->map_count++;
> > +	}
> > +
> > +	if (vp->anon_vma) {
> > +		anon_vma_interval_tree_post_update_vma(vp->vma);
> > +		if (vp->adj_next)
> > +			anon_vma_interval_tree_post_update_vma(vp->adj_next);
> > +		anon_vma_unlock_write(vp->anon_vma);
> > +	}
> > +
> > +	if (vp->file) {
> > +		i_mmap_unlock_write(vp->mapping);
> > +		uprobe_mmap(vp->vma);
> > +
> > +		if (vp->adj_next)
> > +			uprobe_mmap(vp->adj_next);
> > +	}
> > +
> > +	if (vp->remove) {
> > +again:
> > +		vma_mark_detached(vp->remove, true);
> > +		if (vp->file) {
> > +			uprobe_munmap(vp->remove, vp->remove->vm_start,
> > +				      vp->remove->vm_end);
> > +			fput(vp->file);
> > +		}
> > +		if (vp->remove->anon_vma)
> > +			anon_vma_merge(vp->vma, vp->remove);
> > +		mm->map_count--;
> > +		mpol_put(vma_policy(vp->remove));
> > +		if (!vp->remove2)
> > +			WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
> > +		vm_area_free(vp->remove);
> > +
> > +		/*
> > +		 * In mprotect's case 6 (see comments on vma_merge),
> > +		 * we are removing both mid and next vmas
> > +		 */
> > +		if (vp->remove2) {
> > +			vp->remove = vp->remove2;
> > +			vp->remove2 = NULL;
> > +			goto again;
> > +		}
> > +	}
> > +	if (vp->insert && vp->file)
> > +		uprobe_mmap(vp->insert);
> > +	validate_mm(mm);
> > +}
> > +
> > +/*
> > + * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
> > + * @vmi: The vma iterator
> > + * @vma: The starting vm_area_struct
> > + * @mm: The mm_struct
> > + * @start: The aligned start address to munmap.
> > + * @end: The aligned end address to munmap.
> > + * @uf: The userfaultfd list_head
> > + * @unlock: Set to true to drop the mmap_lock.  unlocking only happens on
> > + * success.
> > + *
> > + * Return: 0 on success and drops the lock if so directed, error and leaves the
> > + * lock held otherwise.
> > + */
> > +int
> > +do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +		    struct mm_struct *mm, unsigned long start,
> > +		    unsigned long end, struct list_head *uf, bool unlock)
> > +{
> > +	struct vm_area_struct *prev, *next = NULL;
> > +	struct maple_tree mt_detach;
> > +	int count = 0;
> > +	int error = -ENOMEM;
> > +	unsigned long locked_vm = 0;
> > +	MA_STATE(mas_detach, &mt_detach, 0, 0);
> > +	mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
> > +	mt_on_stack(mt_detach);
> > +
> > +	/*
> > +	 * If we need to split any vma, do it now to save pain later.
> > +	 *
> > +	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
> > +	 * unmapped vm_area_struct will remain in use: so lower split_vma
> > +	 * places tmp vma above, and higher split_vma places tmp vma below.
> > +	 */
> > +
> > +	/* Does it split the first one? */
> > +	if (start > vma->vm_start) {
> > +
> > +		/*
> > +		 * Make sure that map_count on return from munmap() will
> > +		 * not exceed its limit; but let map_count go just above
> > +		 * its limit temporarily, to help free resources as expected.
> > +		 */
> > +		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
> > +			goto map_count_exceeded;
> > +
> > +		error = __split_vma(vmi, vma, start, 1);
> > +		if (error)
> > +			goto start_split_failed;
> > +	}
> > +
> > +	/*
> > +	 * Detach a range of VMAs from the mm. Using next as a temp variable as
> > +	 * it is always overwritten.
> > +	 */
> > +	next = vma;
> > +	do {
> > +		/* Does it split the end? */
> > +		if (next->vm_end > end) {
> > +			error = __split_vma(vmi, next, end, 0);
> > +			if (error)
> > +				goto end_split_failed;
> > +		}
> > +		vma_start_write(next);
> > +		mas_set(&mas_detach, count);
> > +		error = mas_store_gfp(&mas_detach, next, GFP_KERNEL);
> > +		if (error)
> > +			goto munmap_gather_failed;
> > +		vma_mark_detached(next, true);
> > +		if (next->vm_flags & VM_LOCKED)
> > +			locked_vm += vma_pages(next);
> > +
> > +		count++;
> > +		if (unlikely(uf)) {
> > +			/*
> > +			 * If userfaultfd_unmap_prep returns an error the vmas
> > +			 * will remain split, but userland will get a
> > +			 * highly unexpected error anyway. This is no
> > +			 * different than the case where the first of the two
> > +			 * __split_vma fails, but we don't undo the first
> > +			 * split, despite we could. This is unlikely enough
> > +			 * failure that it's not worth optimizing it for.
> > +			 */
> > +			error = userfaultfd_unmap_prep(next, start, end, uf);
> > +
> > +			if (error)
> > +				goto userfaultfd_error;
> > +		}
> > +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> > +		BUG_ON(next->vm_start < start);
> > +		BUG_ON(next->vm_start > end);
> > +#endif
> > +	} for_each_vma_range(*vmi, next, end);
> > +
> > +#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> > +	/* Make sure no VMAs are about to be lost. */
> > +	{
> > +		MA_STATE(test, &mt_detach, 0, 0);
> > +		struct vm_area_struct *vma_mas, *vma_test;
> > +		int test_count = 0;
> > +
> > +		vma_iter_set(vmi, start);
> > +		rcu_read_lock();
> > +		vma_test = mas_find(&test, count - 1);
> > +		for_each_vma_range(*vmi, vma_mas, end) {
> > +			BUG_ON(vma_mas != vma_test);
> > +			test_count++;
> > +			vma_test = mas_next(&test, count - 1);
> > +		}
> > +		rcu_read_unlock();
> > +		BUG_ON(count != test_count);
> > +	}
> > +#endif
> > +
> > +	while (vma_iter_addr(vmi) > start)
> > +		vma_iter_prev_range(vmi);
> > +
> > +	error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
> > +	if (error)
> > +		goto clear_tree_failed;
> > +
> > +	/* Point of no return */
> > +	mm->locked_vm -= locked_vm;
> > +	mm->map_count -= count;
> > +	if (unlock)
> > +		mmap_write_downgrade(mm);
> > +
> > +	prev = vma_iter_prev_range(vmi);
> > +	next = vma_next(vmi);
> > +	if (next)
> > +		vma_iter_prev_range(vmi);
> > +
> > +	/*
> > +	 * We can free page tables without write-locking mmap_lock because VMAs
> > +	 * were isolated before we downgraded mmap_lock.
> > +	 */
> > +	mas_set(&mas_detach, 1);
> > +	unmap_region(mm, &mas_detach, vma, prev, next, start, end, count,
> > +		     !unlock);
> > +	/* Statistics and freeing VMAs */
> > +	mas_set(&mas_detach, 0);
> > +	remove_mt(mm, &mas_detach);
> > +	validate_mm(mm);
> > +	if (unlock)
> > +		mmap_read_unlock(mm);
> > +
> > +	__mt_destroy(&mt_detach);
> > +	return 0;
> > +
> > +clear_tree_failed:
> > +userfaultfd_error:
> > +munmap_gather_failed:
> > +end_split_failed:
> > +	mas_set(&mas_detach, 0);
> > +	mas_for_each(&mas_detach, next, end)
> > +		vma_mark_detached(next, false);
> > +
> > +	__mt_destroy(&mt_detach);
> > +start_split_failed:
> > +map_count_exceeded:
> > +	validate_mm(mm);
> > +	return error;
> > +}
> > +
> > +/*
> > + * do_vmi_munmap() - munmap a given range.
> > + * @vmi: The vma iterator
> > + * @mm: The mm_struct
> > + * @start: The start address to munmap
> > + * @len: The length of the range to munmap
> > + * @uf: The userfaultfd list_head
> > + * @unlock: set to true if the user wants to drop the mmap_lock on success
> > + *
> > + * This function takes a @mas that is either pointing to the previous VMA or set
> > + * to MA_START and sets it up to remove the mapping(s).  The @len will be
> > + * aligned and any arch_unmap work will be preformed.
> > + *
> > + * Return: 0 on success and drops the lock if so directed, error and leaves the
> > + * lock held otherwise.
> > + */
> > +int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
> > +		  unsigned long start, size_t len, struct list_head *uf,
> > +		  bool unlock)
> > +{
> > +	unsigned long end;
> > +	struct vm_area_struct *vma;
> > +
> > +	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
> > +		return -EINVAL;
> > +
> > +	end = start + PAGE_ALIGN(len);
> > +	if (end == start)
> > +		return -EINVAL;
> > +
> > +	/*
> > +	 * Check if memory is sealed before arch_unmap.
> > +	 * Prevent unmapping a sealed VMA.
> > +	 * can_modify_mm assumes we have acquired the lock on MM.
> > +	 */
> > +	if (unlikely(!can_modify_mm(mm, start, end)))
> > +		return -EPERM;
> > +
> > +	 /* arch_unmap() might do unmaps itself.  */
> > +	arch_unmap(mm, start, end);
> > +
> > +	/* Find the first overlapping VMA */
> > +	vma = vma_find(vmi, end);
> > +	if (!vma) {
> > +		if (unlock)
> > +			mmap_write_unlock(mm);
> > +		return 0;
> > +	}
> > +
> > +	return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
> > +}
> > +
> > +/*
> > + * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
> > + * figure out whether that can be merged with its predecessor or its
> > + * successor.  Or both (it neatly fills a hole).
> > + *
> > + * In most cases - when called for mmap, brk or mremap - [addr,end) is
> > + * certain not to be mapped by the time vma_merge is called; but when
> > + * called for mprotect, it is certain to be already mapped (either at
> > + * an offset within prev, or at the start of next), and the flags of
> > + * this area are about to be changed to vm_flags - and the no-change
> > + * case has already been eliminated.
> > + *
> > + * The following mprotect cases have to be considered, where **** is
> > + * the area passed down from mprotect_fixup, never extending beyond one
> > + * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts
> > + * at the same address as **** and is of the same or larger span, and
> > + * NNNN the next vma after ****:
> > + *
> > + *     ****             ****                   ****
> > + *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPCCCCCC
> > + *    cannot merge    might become       might become
> > + *                    PPNNNNNNNNNN       PPPPPPPPPPCC
> > + *    mmap, brk or    case 4 below       case 5 below
> > + *    mremap move:
> > + *                        ****               ****
> > + *                    PPPP    NNNN       PPPPCCCCNNNN
> > + *                    might become       might become
> > + *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
> > + *                    PPPPPPPPNNNN 2 or  PPPPPPPPNNNN 7 or
> > + *                    PPPPNNNNNNNN 3     PPPPNNNNNNNN 8
> > + *
> > + * It is important for case 8 that the vma CCCC overlapping the
> > + * region **** is never going to extended over NNNN. Instead NNNN must
> > + * be extended in region **** and CCCC must be removed. This way in
> > + * all cases where vma_merge succeeds, the moment vma_merge drops the
> > + * rmap_locks, the properties of the merged vma will be already
> > + * correct for the whole merged range. Some of those properties like
> > + * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
> > + * be correct for the whole merged range immediately after the
> > + * rmap_locks are released. Otherwise if NNNN would be removed and
> > + * CCCC would be extended over the NNNN range, remove_migration_ptes
> > + * or other rmap walkers (if working on addresses beyond the "end"
> > + * parameter) may establish ptes with the wrong permissions of CCCC
> > + * instead of the right permissions of NNNN.
> > + *
> > + * In the code below:
> > + * PPPP is represented by *prev
> > + * CCCC is represented by *curr or not represented at all (NULL)
> > + * NNNN is represented by *next or not represented at all (NULL)
> > + * **** is not represented - it will be merged and the vma containing the
> > + *      area is returned, or the function will return NULL
> > + */
> > +static struct vm_area_struct
> > +*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
> > +	   struct vm_area_struct *src, unsigned long addr, unsigned long end,
> > +	   unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
> > +	   struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > +	   struct anon_vma_name *anon_name)
> > +{
> > +	struct mm_struct *mm = src->vm_mm;
> > +	struct anon_vma *anon_vma = src->anon_vma;
> > +	struct file *file = src->vm_file;
> > +	struct vm_area_struct *curr, *next, *res;
> > +	struct vm_area_struct *vma, *adjust, *remove, *remove2;
> > +	struct vm_area_struct *anon_dup = NULL;
> > +	struct vma_prepare vp;
> > +	pgoff_t vma_pgoff;
> > +	int err = 0;
> > +	bool merge_prev = false;
> > +	bool merge_next = false;
> > +	bool vma_expanded = false;
> > +	unsigned long vma_start = addr;
> > +	unsigned long vma_end = end;
> > +	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
> > +	long adj_start = 0;
> > +
> > +	/*
> > +	 * We later require that vma->vm_flags == vm_flags,
> > +	 * so this tests vma->vm_flags & VM_SPECIAL, too.
> > +	 */
> > +	if (vm_flags & VM_SPECIAL)
> > +		return NULL;
> > +
> > +	/* Does the input range span an existing VMA? (cases 5 - 8) */
> > +	curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end);
> > +
> > +	if (!curr ||			/* cases 1 - 4 */
> > +	    end == curr->vm_end)	/* cases 6 - 8, adjacent VMA */
> > +		next = vma_lookup(mm, end);
> > +	else
> > +		next = NULL;		/* case 5 */
> > +
> > +	if (prev) {
> > +		vma_start = prev->vm_start;
> > +		vma_pgoff = prev->vm_pgoff;
> > +
> > +		/* Can we merge the predecessor? */
> > +		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)
> > +		    && can_vma_merge_after(prev, vm_flags, anon_vma, file,
> > +					   pgoff, vm_userfaultfd_ctx, anon_name)) {
> > +			merge_prev = true;
> > +			vma_prev(vmi);
> > +		}
> > +	}
> > +
> > +	/* Can we merge the successor? */
> > +	if (next && mpol_equal(policy, vma_policy(next)) &&
> > +	    can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,
> > +				 vm_userfaultfd_ctx, anon_name)) {
> > +		merge_next = true;
> > +	}
> > +
> > +	/* Verify some invariant that must be enforced by the caller. */
> > +	VM_WARN_ON(prev && addr <= prev->vm_start);
> > +	VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end));
> > +	VM_WARN_ON(addr >= end);
> > +
> > +	if (!merge_prev && !merge_next)
> > +		return NULL; /* Not mergeable. */
> > +
> > +	if (merge_prev)
> > +		vma_start_write(prev);
> > +
> > +	res = vma = prev;
> > +	remove = remove2 = adjust = NULL;
> > +
> > +	/* Can we merge both the predecessor and the successor? */
> > +	if (merge_prev && merge_next &&
> > +	    is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) {
> > +		vma_start_write(next);
> > +		remove = next;				/* case 1 */
> > +		vma_end = next->vm_end;
> > +		err = dup_anon_vma(prev, next, &anon_dup);
> > +		if (curr) {				/* case 6 */
> > +			vma_start_write(curr);
> > +			remove = curr;
> > +			remove2 = next;
> > +			/*
> > +			 * Note that the dup_anon_vma below cannot overwrite err
> > +			 * since the first caller would do nothing unless next
> > +			 * has an anon_vma.
> > +			 */
> > +			if (!next->anon_vma)
> > +				err = dup_anon_vma(prev, curr, &anon_dup);
> > +		}
> > +	} else if (merge_prev) {			/* case 2 */
> > +		if (curr) {
> > +			vma_start_write(curr);
> > +			if (end == curr->vm_end) {	/* case 7 */
> > +				/*
> > +				 * can_vma_merge_after() assumed we would not be
> > +				 * removing prev vma, so it skipped the check
> > +				 * for vm_ops->close, but we are removing curr
> > +				 */
> > +				if (curr->vm_ops && curr->vm_ops->close)
> > +					err = -EINVAL;
> > +				remove = curr;
> > +			} else {			/* case 5 */
> > +				adjust = curr;
> > +				adj_start = (end - curr->vm_start);
> > +			}
> > +			if (!err)
> > +				err = dup_anon_vma(prev, curr, &anon_dup);
> > +		}
> > +	} else { /* merge_next */
> > +		vma_start_write(next);
> > +		res = next;
> > +		if (prev && addr < prev->vm_end) {	/* case 4 */
> > +			vma_start_write(prev);
> > +			vma_end = addr;
> > +			adjust = next;
> > +			adj_start = -(prev->vm_end - addr);
> > +			err = dup_anon_vma(next, prev, &anon_dup);
> > +		} else {
> > +			/*
> > +			 * Note that cases 3 and 8 are the ONLY ones where prev
> > +			 * is permitted to be (but is not necessarily) NULL.
> > +			 */
> > +			vma = next;			/* case 3 */
> > +			vma_start = addr;
> > +			vma_end = next->vm_end;
> > +			vma_pgoff = next->vm_pgoff - pglen;
> > +			if (curr) {			/* case 8 */
> > +				vma_pgoff = curr->vm_pgoff;
> > +				vma_start_write(curr);
> > +				remove = curr;
> > +				err = dup_anon_vma(next, curr, &anon_dup);
> > +			}
> > +		}
> > +	}
> > +
> > +	/* Error in anon_vma clone. */
> > +	if (err)
> > +		goto anon_vma_fail;
> > +
> > +	if (vma_start < vma->vm_start || vma_end > vma->vm_end)
> > +		vma_expanded = true;
> > +
> > +	if (vma_expanded) {
> > +		vma_iter_config(vmi, vma_start, vma_end);
> > +	} else {
> > +		vma_iter_config(vmi, adjust->vm_start + adj_start,
> > +				adjust->vm_end);
> > +	}
> > +
> > +	if (vma_iter_prealloc(vmi, vma))
> > +		goto prealloc_fail;
> > +
> > +	init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
> > +	VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
> > +		   vp.anon_vma != adjust->anon_vma);
> > +
> > +	vma_prepare(&vp);
> > +	vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start);
> > +	vma_set_range(vma, vma_start, vma_end, vma_pgoff);
> > +
> > +	if (vma_expanded)
> > +		vma_iter_store(vmi, vma);
> > +
> > +	if (adj_start) {
> > +		adjust->vm_start += adj_start;
> > +		adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
> > +		if (adj_start < 0) {
> > +			WARN_ON(vma_expanded);
> > +			vma_iter_store(vmi, next);
> > +		}
> > +	}
> > +
> > +	vma_complete(&vp, vmi, mm);
> > +	khugepaged_enter_vma(res, vm_flags);
> > +	return res;
> > +
> > +prealloc_fail:
> > +	if (anon_dup)
> > +		unlink_anon_vmas(anon_dup);
> > +
> > +anon_vma_fail:
> > +	vma_iter_set(vmi, addr);
> > +	vma_iter_load(vmi);
> > +	return NULL;
> > +}
> > +
> > +/*
> > + * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
> > + * context and anonymous VMA name within the range [start, end).
> > + *
> > + * As a result, we might be able to merge the newly modified VMA range with an
> > + * adjacent VMA with identical properties.
> > + *
> > + * If no merge is possible and the range does not span the entirety of the VMA,
> > + * we then need to split the VMA to accommodate the change.
> > + *
> > + * The function returns either the merged VMA, the original VMA if a split was
> > + * required instead, or an error if the split failed.
> > + */
> > +struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> > +				  struct vm_area_struct *prev,
> > +				  struct vm_area_struct *vma,
> > +				  unsigned long start, unsigned long end,
> > +				  unsigned long vm_flags,
> > +				  struct mempolicy *policy,
> > +				  struct vm_userfaultfd_ctx uffd_ctx,
> > +				  struct anon_vma_name *anon_name)
> > +{
> > +	pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> > +	struct vm_area_struct *merged;
> > +
> > +	merged = vma_merge(vmi, prev, vma, start, end, vm_flags,
> > +			   pgoff, policy, uffd_ctx, anon_name);
> > +	if (merged)
> > +		return merged;
> > +
> > +	if (vma->vm_start < start) {
> > +		int err = split_vma(vmi, vma, start, 1);
> > +
> > +		if (err)
> > +			return ERR_PTR(err);
> > +	}
> > +
> > +	if (vma->vm_end > end) {
> > +		int err = split_vma(vmi, vma, end, 0);
> > +
> > +		if (err)
> > +			return ERR_PTR(err);
> > +	}
> > +
> > +	return vma;
> > +}
> > +
> > +/*
> > + * Attempt to merge a newly mapped VMA with those adjacent to it. The caller
> > + * must ensure that [start, end) does not overlap any existing VMA.
> > + */
> > +struct vm_area_struct
> > +*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
> > +		   struct vm_area_struct *vma, unsigned long start,
> > +		   unsigned long end, pgoff_t pgoff)
> > +{
> > +	return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,
> > +			 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> > +}
> > +
> > +/*
> > + * Expand vma by delta bytes, potentially merging with an immediately adjacent
> > + * VMA with identical properties.
> > + */
> > +struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> > +					struct vm_area_struct *vma,
> > +					unsigned long delta)
> > +{
> > +	pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);
> > +
> > +	/* vma is specified as prev, so case 1 or 2 will apply. */
> > +	return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,
> > +			 vma->vm_flags, pgoff, vma_policy(vma),
> > +			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> > +}
> > +
> > +void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)
> > +{
> > +	vb->count = 0;
> > +}
> > +
> > +static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb)
> > +{
> > +	struct address_space *mapping;
> > +	int i;
> > +
> > +	mapping = vb->vmas[0]->vm_file->f_mapping;
> > +	i_mmap_lock_write(mapping);
> > +	for (i = 0; i < vb->count; i++) {
> > +		VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping);
> > +		__remove_shared_vm_struct(vb->vmas[i], mapping);
> > +	}
> > +	i_mmap_unlock_write(mapping);
> > +
> > +	unlink_file_vma_batch_init(vb);
> > +}
> > +
> > +void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
> > +			       struct vm_area_struct *vma)
> > +{
> > +	if (vma->vm_file == NULL)
> > +		return;
> > +
> > +	if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) ||
> > +	    vb->count == ARRAY_SIZE(vb->vmas))
> > +		unlink_file_vma_batch_process(vb);
> > +
> > +	vb->vmas[vb->count] = vma;
> > +	vb->count++;
> > +}
> > +
> > +void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb)
> > +{
> > +	if (vb->count > 0)
> > +		unlink_file_vma_batch_process(vb);
> > +}
> > +
> > +/*
> > + * Unlink a file-based vm structure from its interval tree, to hide
> > + * vma from rmap and vmtruncate before freeing its page tables.
> > + */
> > +void unlink_file_vma(struct vm_area_struct *vma)
> > +{
> > +	struct file *file = vma->vm_file;
> > +
> > +	if (file) {
> > +		struct address_space *mapping = file->f_mapping;
> > +
> > +		i_mmap_lock_write(mapping);
> > +		__remove_shared_vm_struct(vma, mapping);
> > +		i_mmap_unlock_write(mapping);
> > +	}
> > +}
> > +
> > +void vma_link_file(struct vm_area_struct *vma)
> > +{
> > +	struct file *file = vma->vm_file;
> > +	struct address_space *mapping;
> > +
> > +	if (file) {
> > +		mapping = file->f_mapping;
> > +		i_mmap_lock_write(mapping);
> > +		__vma_link_file(vma, mapping);
> > +		i_mmap_unlock_write(mapping);
> > +	}
> > +}
> > +
> > +int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
> > +{
> > +	VMA_ITERATOR(vmi, mm, 0);
> > +
> > +	vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
> > +	if (vma_iter_prealloc(&vmi, vma))
> > +		return -ENOMEM;
> > +
> > +	vma_start_write(vma);
> > +	vma_iter_store(&vmi, vma);
> > +	vma_link_file(vma);
> > +	mm->map_count++;
> > +	validate_mm(mm);
> > +	return 0;
> > +}
> > +
> > +/*
> > + * Copy the vma structure to a new location in the same mm,
> > + * prior to moving page table entries, to effect an mremap move.
> > + */
> > +struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> > +	unsigned long addr, unsigned long len, pgoff_t pgoff,
> > +	bool *need_rmap_locks)
> > +{
> > +	struct vm_area_struct *vma = *vmap;
> > +	unsigned long vma_start = vma->vm_start;
> > +	struct mm_struct *mm = vma->vm_mm;
> > +	struct vm_area_struct *new_vma, *prev;
> > +	bool faulted_in_anon_vma = true;
> > +	VMA_ITERATOR(vmi, mm, addr);
> > +
> > +	/*
> > +	 * If anonymous vma has not yet been faulted, update new pgoff
> > +	 * to match new location, to increase its chance of merging.
> > +	 */
> > +	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
> > +		pgoff = addr >> PAGE_SHIFT;
> > +		faulted_in_anon_vma = false;
> > +	}
> > +
> > +	new_vma = find_vma_prev(mm, addr, &prev);
> > +	if (new_vma && new_vma->vm_start < addr + len)
> > +		return NULL;	/* should never get here */
> > +
> > +	new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff);
> > +	if (new_vma) {
> > +		/*
> > +		 * Source vma may have been merged into new_vma
> > +		 */
> > +		if (unlikely(vma_start >= new_vma->vm_start &&
> > +			     vma_start < new_vma->vm_end)) {
> > +			/*
> > +			 * The only way we can get a vma_merge with
> > +			 * self during an mremap is if the vma hasn't
> > +			 * been faulted in yet and we were allowed to
> > +			 * reset the dst vma->vm_pgoff to the
> > +			 * destination address of the mremap to allow
> > +			 * the merge to happen. mremap must change the
> > +			 * vm_pgoff linearity between src and dst vmas
> > +			 * (in turn preventing a vma_merge) to be
> > +			 * safe. It is only safe to keep the vm_pgoff
> > +			 * linear if there are no pages mapped yet.
> > +			 */
> > +			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
> > +			*vmap = vma = new_vma;
> > +		}
> > +		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
> > +	} else {
> > +		new_vma = vm_area_dup(vma);
> > +		if (!new_vma)
> > +			goto out;
> > +		vma_set_range(new_vma, addr, addr + len, pgoff);
> > +		if (vma_dup_policy(vma, new_vma))
> > +			goto out_free_vma;
> > +		if (anon_vma_clone(new_vma, vma))
> > +			goto out_free_mempol;
> > +		if (new_vma->vm_file)
> > +			get_file(new_vma->vm_file);
> > +		if (new_vma->vm_ops && new_vma->vm_ops->open)
> > +			new_vma->vm_ops->open(new_vma);
> > +		if (vma_link(mm, new_vma))
> > +			goto out_vma_link;
> > +		*need_rmap_locks = false;
> > +	}
> > +	return new_vma;
> > +
> > +out_vma_link:
> > +	if (new_vma->vm_ops && new_vma->vm_ops->close)
> > +		new_vma->vm_ops->close(new_vma);
> > +
> > +	if (new_vma->vm_file)
> > +		fput(new_vma->vm_file);
> > +
> > +	unlink_anon_vmas(new_vma);
> > +out_free_mempol:
> > +	mpol_put(vma_policy(new_vma));
> > +out_free_vma:
> > +	vm_area_free(new_vma);
> > +out:
> > +	return NULL;
> > +}
> > +
> > +/*
> > + * Rough compatibility check to quickly see if it's even worth looking
> > + * at sharing an anon_vma.
> > + *
> > + * They need to have the same vm_file, and the flags can only differ
> > + * in things that mprotect may change.
> > + *
> > + * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
> > + * we can merge the two vma's. For example, we refuse to merge a vma if
> > + * there is a vm_ops->close() function, because that indicates that the
> > + * driver is doing some kind of reference counting. But that doesn't
> > + * really matter for the anon_vma sharing case.
> > + */
> > +static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
> > +{
> > +	return a->vm_end == b->vm_start &&
> > +		mpol_equal(vma_policy(a), vma_policy(b)) &&
> > +		a->vm_file == b->vm_file &&
> > +		!((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
> > +		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
> > +}
> > +
> > +/*
> > + * Do some basic sanity checking to see if we can re-use the anon_vma
> > + * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
> > + * the same as 'old', the other will be the new one that is trying
> > + * to share the anon_vma.
> > + *
> > + * NOTE! This runs with mmap_lock held for reading, so it is possible that
> > + * the anon_vma of 'old' is concurrently in the process of being set up
> > + * by another page fault trying to merge _that_. But that's ok: if it
> > + * is being set up, that automatically means that it will be a singleton
> > + * acceptable for merging, so we can do all of this optimistically. But
> > + * we do that READ_ONCE() to make sure that we never re-load the pointer.
> > + *
> > + * IOW: that the "list_is_singular()" test on the anon_vma_chain only
> > + * matters for the 'stable anon_vma' case (ie the thing we want to avoid
> > + * is to return an anon_vma that is "complex" due to having gone through
> > + * a fork).
> > + *
> > + * We also make sure that the two vma's are compatible (adjacent,
> > + * and with the same memory policies). That's all stable, even with just
> > + * a read lock on the mmap_lock.
> > + */
> > +static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old,
> > +					  struct vm_area_struct *a,
> > +					  struct vm_area_struct *b)
> > +{
> > +	if (anon_vma_compatible(a, b)) {
> > +		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
> > +
> > +		if (anon_vma && list_is_singular(&old->anon_vma_chain))
> > +			return anon_vma;
> > +	}
> > +	return NULL;
> > +}
> > +
> > +/*
> > + * find_mergeable_anon_vma is used by anon_vma_prepare, to check
> > + * neighbouring vmas for a suitable anon_vma, before it goes off
> > + * to allocate a new anon_vma.  It checks because a repetitive
> > + * sequence of mprotects and faults may otherwise lead to distinct
> > + * anon_vmas being allocated, preventing vma merge in subsequent
> > + * mprotect.
> > + */
> > +struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
> > +{
> > +	struct anon_vma *anon_vma = NULL;
> > +	struct vm_area_struct *prev, *next;
> > +	VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end);
> > +
> > +	/* Try next first. */
> > +	next = vma_iter_load(&vmi);
> > +	if (next) {
> > +		anon_vma = reusable_anon_vma(next, vma, next);
> > +		if (anon_vma)
> > +			return anon_vma;
> > +	}
> > +
> > +	prev = vma_prev(&vmi);
> > +	VM_BUG_ON_VMA(prev != vma, vma);
> > +	prev = vma_prev(&vmi);
> > +	/* Try prev next. */
> > +	if (prev)
> > +		anon_vma = reusable_anon_vma(prev, prev, vma);
> > +
> > +	/*
> > +	 * We might reach here with anon_vma == NULL if we can't find
> > +	 * any reusable anon_vma.
> > +	 * There's no absolute need to look only at touching neighbours:
> > +	 * we could search further afield for "compatible" anon_vmas.
> > +	 * But it would probably just be a waste of time searching,
> > +	 * or lead to too many vmas hanging off the same anon_vma.
> > +	 * We're trying to allow mprotect remerging later on,
> > +	 * not trying to minimize memory used for anon_vmas.
> > +	 */
> > +	return anon_vma;
> > +}
> > +
> > +static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
> > +{
> > +	return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
> > +}
> > +
> > +static bool vma_is_shared_writable(struct vm_area_struct *vma)
> > +{
> > +	return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
> > +		(VM_WRITE | VM_SHARED);
> > +}
> > +
> > +static bool vma_fs_can_writeback(struct vm_area_struct *vma)
> > +{
> > +	/* No managed pages to writeback. */
> > +	if (vma->vm_flags & VM_PFNMAP)
> > +		return false;
> > +
> > +	return vma->vm_file && vma->vm_file->f_mapping &&
> > +		mapping_can_writeback(vma->vm_file->f_mapping);
> > +}
> > +
> > +/*
> > + * Does this VMA require the underlying folios to have their dirty state
> > + * tracked?
> > + */
> > +bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
> > +{
> > +	/* Only shared, writable VMAs require dirty tracking. */
> > +	if (!vma_is_shared_writable(vma))
> > +		return false;
> > +
> > +	/* Does the filesystem need to be notified? */
> > +	if (vm_ops_needs_writenotify(vma->vm_ops))
> > +		return true;
> > +
> > +	/*
> > +	 * Even if the filesystem doesn't indicate a need for writenotify, if it
> > +	 * can writeback, dirty tracking is still required.
> > +	 */
> > +	return vma_fs_can_writeback(vma);
> > +}
> > +
> > +/*
> > + * Some shared mappings will want the pages marked read-only
> > + * to track write events. If so, we'll downgrade vm_page_prot
> > + * to the private version (using protection_map[] without the
> > + * VM_SHARED bit).
> > + */
> > +bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
> > +{
> > +	/* If it was private or non-writable, the write bit is already clear */
> > +	if (!vma_is_shared_writable(vma))
> > +		return false;
> > +
> > +	/* The backer wishes to know when pages are first written to? */
> > +	if (vm_ops_needs_writenotify(vma->vm_ops))
> > +		return true;
> > +
> > +	/* The open routine did something to the protections that pgprot_modify
> > +	 * won't preserve? */
> > +	if (pgprot_val(vm_page_prot) !=
> > +	    pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
> > +		return false;
> > +
> > +	/*
> > +	 * Do we need to track softdirty? hugetlb does not support softdirty
> > +	 * tracking yet.
> > +	 */
> > +	if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
> > +		return true;
> > +
> > +	/* Do we need write faults for uffd-wp tracking? */
> > +	if (userfaultfd_wp(vma))
> > +		return true;
> > +
> > +	/* Can the mapping track the dirty pages? */
> > +	return vma_fs_can_writeback(vma);
> > +}
> > +
> > +unsigned long count_vma_pages_range(struct mm_struct *mm,
> > +				    unsigned long addr, unsigned long end)
> > +{
> > +	VMA_ITERATOR(vmi, mm, addr);
> > +	struct vm_area_struct *vma;
> > +	unsigned long nr_pages = 0;
> > +
> > +	for_each_vma_range(vmi, vma, end) {
> > +		unsigned long vm_start = max(addr, vma->vm_start);
> > +		unsigned long vm_end = min(end, vma->vm_end);
> > +
> > +		nr_pages += PHYS_PFN(vm_end - vm_start);
> > +	}
> > +
> > +	return nr_pages;
> > +}
> > +
> > +static DEFINE_MUTEX(mm_all_locks_mutex);
> > +
> > +static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
> > +{
> > +	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> > +		/*
> > +		 * The LSB of head.next can't change from under us
> > +		 * because we hold the mm_all_locks_mutex.
> > +		 */
> > +		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
> > +		/*
> > +		 * We can safely modify head.next after taking the
> > +		 * anon_vma->root->rwsem. If some other vma in this mm shares
> > +		 * the same anon_vma we won't take it again.
> > +		 *
> > +		 * No need of atomic instructions here, head.next
> > +		 * can't change from under us thanks to the
> > +		 * anon_vma->root->rwsem.
> > +		 */
> > +		if (__test_and_set_bit(0, (unsigned long *)
> > +				       &anon_vma->root->rb_root.rb_root.rb_node))
> > +			BUG();
> > +	}
> > +}
> > +
> > +static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
> > +{
> > +	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> > +		/*
> > +		 * AS_MM_ALL_LOCKS can't change from under us because
> > +		 * we hold the mm_all_locks_mutex.
> > +		 *
> > +		 * Operations on ->flags have to be atomic because
> > +		 * even if AS_MM_ALL_LOCKS is stable thanks to the
> > +		 * mm_all_locks_mutex, there may be other cpus
> > +		 * changing other bitflags in parallel to us.
> > +		 */
> > +		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
> > +			BUG();
> > +		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
> > +	}
> > +}
> > +
> > +/*
> > + * This operation locks against the VM for all pte/vma/mm related
> > + * operations that could ever happen on a certain mm. This includes
> > + * vmtruncate, try_to_unmap, and all page faults.
> > + *
> > + * The caller must take the mmap_lock in write mode before calling
> > + * mm_take_all_locks(). The caller isn't allowed to release the
> > + * mmap_lock until mm_drop_all_locks() returns.
> > + *
> > + * mmap_lock in write mode is required in order to block all operations
> > + * that could modify pagetables and free pages without need of
> > + * altering the vma layout. It's also needed in write mode to avoid new
> > + * anon_vmas to be associated with existing vmas.
> > + *
> > + * A single task can't take more than one mm_take_all_locks() in a row
> > + * or it would deadlock.
> > + *
> > + * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
> > + * mapping->flags avoid to take the same lock twice, if more than one
> > + * vma in this mm is backed by the same anon_vma or address_space.
> > + *
> > + * We take locks in following order, accordingly to comment at beginning
> > + * of mm/rmap.c:
> > + *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
> > + *     hugetlb mapping);
> > + *   - all vmas marked locked
> > + *   - all i_mmap_rwsem locks;
> > + *   - all anon_vma->rwseml
> > + *
> > + * We can take all locks within these types randomly because the VM code
> > + * doesn't nest them and we protected from parallel mm_take_all_locks() by
> > + * mm_all_locks_mutex.
> > + *
> > + * mm_take_all_locks() and mm_drop_all_locks are expensive operations
> > + * that may have to take thousand of locks.
> > + *
> > + * mm_take_all_locks() can fail if it's interrupted by signals.
> > + */
> > +int mm_take_all_locks(struct mm_struct *mm)
> > +{
> > +	struct vm_area_struct *vma;
> > +	struct anon_vma_chain *avc;
> > +	VMA_ITERATOR(vmi, mm, 0);
> > +
> > +	mmap_assert_write_locked(mm);
> > +
> > +	mutex_lock(&mm_all_locks_mutex);
> > +
> > +	/*
> > +	 * vma_start_write() does not have a complement in mm_drop_all_locks()
> > +	 * because vma_start_write() is always asymmetrical; it marks a VMA as
> > +	 * being written to until mmap_write_unlock() or mmap_write_downgrade()
> > +	 * is reached.
> > +	 */
> > +	for_each_vma(vmi, vma) {
> > +		if (signal_pending(current))
> > +			goto out_unlock;
> > +		vma_start_write(vma);
> > +	}
> > +
> > +	vma_iter_init(&vmi, mm, 0);
> > +	for_each_vma(vmi, vma) {
> > +		if (signal_pending(current))
> > +			goto out_unlock;
> > +		if (vma->vm_file && vma->vm_file->f_mapping &&
> > +				is_vm_hugetlb_page(vma))
> > +			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> > +	}
> > +
> > +	vma_iter_init(&vmi, mm, 0);
> > +	for_each_vma(vmi, vma) {
> > +		if (signal_pending(current))
> > +			goto out_unlock;
> > +		if (vma->vm_file && vma->vm_file->f_mapping &&
> > +				!is_vm_hugetlb_page(vma))
> > +			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> > +	}
> > +
> > +	vma_iter_init(&vmi, mm, 0);
> > +	for_each_vma(vmi, vma) {
> > +		if (signal_pending(current))
> > +			goto out_unlock;
> > +		if (vma->anon_vma)
> > +			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > +				vm_lock_anon_vma(mm, avc->anon_vma);
> > +	}
> > +
> > +	return 0;
> > +
> > +out_unlock:
> > +	mm_drop_all_locks(mm);
> > +	return -EINTR;
> > +}
> > +
> > +static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
> > +{
> > +	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> > +		/*
> > +		 * The LSB of head.next can't change to 0 from under
> > +		 * us because we hold the mm_all_locks_mutex.
> > +		 *
> > +		 * We must however clear the bitflag before unlocking
> > +		 * the vma so the users using the anon_vma->rb_root will
> > +		 * never see our bitflag.
> > +		 *
> > +		 * No need of atomic instructions here, head.next
> > +		 * can't change from under us until we release the
> > +		 * anon_vma->root->rwsem.
> > +		 */
> > +		if (!__test_and_clear_bit(0, (unsigned long *)
> > +					  &anon_vma->root->rb_root.rb_root.rb_node))
> > +			BUG();
> > +		anon_vma_unlock_write(anon_vma);
> > +	}
> > +}
> > +
> > +static void vm_unlock_mapping(struct address_space *mapping)
> > +{
> > +	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> > +		/*
> > +		 * AS_MM_ALL_LOCKS can't change to 0 from under us
> > +		 * because we hold the mm_all_locks_mutex.
> > +		 */
> > +		i_mmap_unlock_write(mapping);
> > +		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
> > +					&mapping->flags))
> > +			BUG();
> > +	}
> > +}
> > +
> > +/*
> > + * The mmap_lock cannot be released by the caller until
> > + * mm_drop_all_locks() returns.
> > + */
> > +void mm_drop_all_locks(struct mm_struct *mm)
> > +{
> > +	struct vm_area_struct *vma;
> > +	struct anon_vma_chain *avc;
> > +	VMA_ITERATOR(vmi, mm, 0);
> > +
> > +	mmap_assert_write_locked(mm);
> > +	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
> > +
> > +	for_each_vma(vmi, vma) {
> > +		if (vma->anon_vma)
> > +			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> > +				vm_unlock_anon_vma(avc->anon_vma);
> > +		if (vma->vm_file && vma->vm_file->f_mapping)
> > +			vm_unlock_mapping(vma->vm_file->f_mapping);
> > +	}
> > +
> > +	mutex_unlock(&mm_all_locks_mutex);
> > +}
> > diff --git a/mm/vma.h b/mm/vma.h
> > new file mode 100644
> > index 000000000000..15d82dbb7213
> > --- /dev/null
> > +++ b/mm/vma.h
> > @@ -0,0 +1,356 @@
> > +/* SPDX-License-Identifier: GPL-2.0-or-later */
> > +/*
> > + * vma.h
> > + *
> > + * Core VMA manipulation API implemented in vma.c.
> > + */
> > +#ifndef __MM_VMA_H
> > +#define __MM_VMA_H
> > +
> > +/*
> > + * VMA lock generalization
> > + */
> > +struct vma_prepare {
> > +	struct vm_area_struct *vma;
> > +	struct vm_area_struct *adj_next;
> > +	struct file *file;
> > +	struct address_space *mapping;
> > +	struct anon_vma *anon_vma;
> > +	struct vm_area_struct *insert;
> > +	struct vm_area_struct *remove;
> > +	struct vm_area_struct *remove2;
> > +};
> > +
> > +struct unlink_vma_file_batch {
> > +	int count;
> > +	struct vm_area_struct *vmas[8];
> > +};
> > +
> > +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> > +void validate_mm(struct mm_struct *mm);
> > +#else
> > +#define validate_mm(mm) do { } while (0)
> > +#endif
> > +
> > +/* Required for expand_downwards(). */
> > +void anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma);
> > +
> > +/* Required for expand_downwards(). */
> > +void anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma);
> > +
> > +/* Required for do_brk_flags(). */
> > +void vma_prepare(struct vma_prepare *vp);
> > +
> > +/* Required for do_brk_flags(). */
> > +void init_vma_prep(struct vma_prepare *vp,
> > +		   struct vm_area_struct *vma);
> > +
> > +/* Required for do_brk_flags(). */
> > +void vma_complete(struct vma_prepare *vp,
> > +		  struct vma_iterator *vmi, struct mm_struct *mm);
> > +
> > +int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +	       unsigned long start, unsigned long end, pgoff_t pgoff,
> > +	       struct vm_area_struct *next);
> > +
> > +int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +	       unsigned long start, unsigned long end, pgoff_t pgoff);
> > +
> > +int
> > +do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
> > +		    struct mm_struct *mm, unsigned long start,
> > +		    unsigned long end, struct list_head *uf, bool unlock);
> > +
> > +int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
> > +		  unsigned long start, size_t len, struct list_head *uf,
> > +		  bool unlock);
> > +
> > +void remove_vma(struct vm_area_struct *vma, bool unreachable);
> > +
> > +void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> > +		struct vm_area_struct *vma, struct vm_area_struct *prev,
> > +		struct vm_area_struct *next, unsigned long start,
> > +		unsigned long end, unsigned long tree_end, bool mm_wr_locked);
> > +
> > +/* Required by mmap_region(). */
> > +bool
> > +can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
> > +		struct anon_vma *anon_vma, struct file *file,
> > +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > +		struct anon_vma_name *anon_name);
> > +
> > +/* Required by mmap_region() and do_brk_flags(). */
> > +bool
> > +can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
> > +		struct anon_vma *anon_vma, struct file *file,
> > +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> > +		struct anon_vma_name *anon_name);
> > +
> > +struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> > +				  struct vm_area_struct *prev,
> > +				  struct vm_area_struct *vma,
> > +				  unsigned long start, unsigned long end,
> > +				  unsigned long vm_flags,
> > +				  struct mempolicy *policy,
> > +				  struct vm_userfaultfd_ctx uffd_ctx,
> > +				  struct anon_vma_name *anon_name);
> > +
> > +/* We are about to modify the VMA's flags. */
> > +static inline struct vm_area_struct
> > +*vma_modify_flags(struct vma_iterator *vmi,
> > +		  struct vm_area_struct *prev,
> > +		  struct vm_area_struct *vma,
> > +		  unsigned long start, unsigned long end,
> > +		  unsigned long new_flags)
> > +{
> > +	return vma_modify(vmi, prev, vma, start, end, new_flags,
> > +			  vma_policy(vma), vma->vm_userfaultfd_ctx,
> > +			  anon_vma_name(vma));
> > +}
> > +
> > +/* We are about to modify the VMA's flags and/or anon_name. */
> > +static inline struct vm_area_struct
> > +*vma_modify_flags_name(struct vma_iterator *vmi,
> > +		       struct vm_area_struct *prev,
> > +		       struct vm_area_struct *vma,
> > +		       unsigned long start,
> > +		       unsigned long end,
> > +		       unsigned long new_flags,
> > +		       struct anon_vma_name *new_name)
> > +{
> > +	return vma_modify(vmi, prev, vma, start, end, new_flags,
> > +			  vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
> > +}
> > +
> > +/* We are about to modify the VMA's memory policy. */
> > +static inline struct vm_area_struct
> > +*vma_modify_policy(struct vma_iterator *vmi,
> > +		   struct vm_area_struct *prev,
> > +		   struct vm_area_struct *vma,
> > +		   unsigned long start, unsigned long end,
> > +		   struct mempolicy *new_pol)
> > +{
> > +	return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
> > +			  new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> > +}
> > +
> > +/* We are about to modify the VMA's flags and/or uffd context. */
> > +static inline struct vm_area_struct
> > +*vma_modify_flags_uffd(struct vma_iterator *vmi,
> > +		       struct vm_area_struct *prev,
> > +		       struct vm_area_struct *vma,
> > +		       unsigned long start, unsigned long end,
> > +		       unsigned long new_flags,
> > +		       struct vm_userfaultfd_ctx new_ctx)
> > +{
> > +	return vma_modify(vmi, prev, vma, start, end, new_flags,
> > +			  vma_policy(vma), new_ctx, anon_vma_name(vma));
> > +}
> > +
> > +struct vm_area_struct
> > +*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
> > +		   struct vm_area_struct *vma, unsigned long start,
> > +		   unsigned long end, pgoff_t pgoff);
> > +
> > +struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> > +					struct vm_area_struct *vma,
> > +					unsigned long delta);
> > +
> > +void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
> > +
> > +void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb);
> > +
> > +void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
> > +			       struct vm_area_struct *vma);
> > +
> > +void unlink_file_vma(struct vm_area_struct *vma);
> > +
> > +void vma_link_file(struct vm_area_struct *vma);
> > +
> > +int vma_link(struct mm_struct *mm, struct vm_area_struct *vma);
> > +
> > +struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> > +	unsigned long addr, unsigned long len, pgoff_t pgoff,
> > +	bool *need_rmap_locks);
> > +
> > +struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
> > +
> > +bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
> > +bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
> > +
> > +int mm_take_all_locks(struct mm_struct *mm);
> > +void mm_drop_all_locks(struct mm_struct *mm);
> > +unsigned long count_vma_pages_range(struct mm_struct *mm,
> > +				    unsigned long addr, unsigned long end);
> > +
> > +static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
> > +{
> > +	/*
> > +	 * We want to check manually if we can change individual PTEs writable
> > +	 * if we can't do that automatically for all PTEs in a mapping. For
> > +	 * private mappings, that's always the case when we have write
> > +	 * permissions as we properly have to handle COW.
> > +	 */
> > +	if (vma->vm_flags & VM_SHARED)
> > +		return vma_wants_writenotify(vma, vma->vm_page_prot);
> > +	return !!(vma->vm_flags & VM_WRITE);
> > +}
> > +
> > +static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
> > +{
> > +	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
> > +}
> > +
> > +static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
> > +						    unsigned long min)
> > +{
> > +	return mas_prev(&vmi->mas, min);
> > +}
> > +
> > +static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
> > +			struct vm_area_struct *vma, gfp_t gfp)
> > +{
> > +	if (vmi->mas.status != ma_start &&
> > +	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> > +		vma_iter_invalidate(vmi);
> > +
> > +	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> > +	mas_store_gfp(&vmi->mas, vma, gfp);
> > +	if (unlikely(mas_is_err(&vmi->mas)))
> > +		return -ENOMEM;
> > +
> > +	return 0;
> > +}
> > +
> > +
> > +/*
> > + * These three helpers classifies VMAs for virtual memory accounting.
> > + */
> > +
> > +/*
> > + * Executable code area - executable, not writable, not stack
> > + */
> > +static inline bool is_exec_mapping(vm_flags_t flags)
> > +{
> > +	return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
> > +}
> > +
> > +/*
> > + * Stack area (including shadow stacks)
> > + *
> > + * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
> > + * do_mmap() forbids all other combinations.
> > + */
> > +static inline bool is_stack_mapping(vm_flags_t flags)
> > +{
> > +	return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
> > +}
> > +
> > +/*
> > + * Data area - private, writable, not stack
> > + */
> > +static inline bool is_data_mapping(vm_flags_t flags)
> > +{
> > +	return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
> > +}
> > +
> > +
> > +static inline void vma_iter_config(struct vma_iterator *vmi,
> > +		unsigned long index, unsigned long last)
> > +{
> > +	__mas_set_range(&vmi->mas, index, last - 1);
> > +}
> > +
> > +static inline void vma_iter_reset(struct vma_iterator *vmi)
> > +{
> > +	mas_reset(&vmi->mas);
> > +}
> > +
> > +static inline
> > +struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
> > +{
> > +	return mas_prev_range(&vmi->mas, min);
> > +}
> > +
> > +static inline
> > +struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
> > +{
> > +	return mas_next_range(&vmi->mas, max);
> > +}
> > +
> > +static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
> > +				       unsigned long max, unsigned long size)
> > +{
> > +	return mas_empty_area(&vmi->mas, min, max - 1, size);
> > +}
> > +
> > +static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
> > +					unsigned long max, unsigned long size)
> > +{
> > +	return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
> > +}
> > +
> > +/*
> > + * VMA Iterator functions shared between nommu and mmap
> > + */
> > +static inline int vma_iter_prealloc(struct vma_iterator *vmi,
> > +		struct vm_area_struct *vma)
> > +{
> > +	return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
> > +}
> > +
> > +static inline void vma_iter_clear(struct vma_iterator *vmi)
> > +{
> > +	mas_store_prealloc(&vmi->mas, NULL);
> > +}
> > +
> > +static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
> > +{
> > +	return mas_walk(&vmi->mas);
> > +}
> > +
> > +/* Store a VMA with preallocated memory */
> > +static inline void vma_iter_store(struct vma_iterator *vmi,
> > +				  struct vm_area_struct *vma)
> > +{
> > +
> > +#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> > +	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> > +			vmi->mas.index > vma->vm_start)) {
> > +		pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
> > +			vmi->mas.index, vma->vm_start, vma->vm_start,
> > +			vma->vm_end, vmi->mas.index, vmi->mas.last);
> > +	}
> > +	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> > +			vmi->mas.last <  vma->vm_start)) {
> > +		pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
> > +		       vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
> > +		       vmi->mas.index, vmi->mas.last);
> > +	}
> > +#endif
> > +
> > +	if (vmi->mas.status != ma_start &&
> > +	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> > +		vma_iter_invalidate(vmi);
> > +
> > +	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> > +	mas_store_prealloc(&vmi->mas, vma);
> > +}
> > +
> > +static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
> > +{
> > +	return vmi->mas.index;
> > +}
> > +
> > +static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
> > +{
> > +	return vmi->mas.last + 1;
> > +}
> > +
> > +static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
> > +				      unsigned long count)
> > +{
> > +	return mas_expected_entries(&vmi->mas, count);
> > +}
> > +
> > +#endif	/* __MM_VMA_H */
> > diff --git a/mm/vma_internal.h b/mm/vma_internal.h
> > new file mode 100644
> > index 000000000000..51b2010f30c0
> > --- /dev/null
> > +++ b/mm/vma_internal.h
> > @@ -0,0 +1,143 @@
> > +/* SPDX-License-Identifier: GPL-2.0-or-later */
> > +/*
> > + * vma_internal.h
> > + *
> > + * Headers required by vma.c, which can be substituted accordingly when testing
> > + * VMA functionality.
> > + */
> > +
>
> This should probably have header guards for the testing side?
>
> > +/* For fundamental mm types and VMA_ITERATOR(),
> > + * tlb_gather_mmu(), tlb_finish_mmu().
> > + */
> > +#include <linux/mm_types.h>
> > +
> > +/* For mapping_can_writeback(). */
> > +#include <linux/backing-dev.h>
> > +
> > +/*
> > + * For test_bit(), test_and_set_bit(), __test_and_set_bit(),
> > + * test_and_clear_bit().
> > + */
> > +#include <linux/bitops.h>
> > +
> > +/* For WARN_ON(), WARN_ON_ONCE(), BUG_ON(). */
> > +#include <linux/bug.h>
> > +
> > +/* For ERR_PTR(). */
> > +#include <linux/err.h>
> > +
> > +/* For fput(). */
> > +#include <linux/file.h>
> > +
> > +/*
> > + * For get_file(), mapping_unmap_writable(), i_mmap_lock_write(),
> > + * i_mmap_unlock_write().
> > + */
> > +#include <linux/fs.h>
> > +
> > +/* For vma_adjust_trans_huge(). */
> > +#include <linux/huge_mm.h>
> > +
> > +/* For is_vm_hugetlb_page(). */
> > +#include <linux/hugetlb_inline.h>
> > +
> > +/* For might_sleep(). */
> > +#include <linux/kernel.h>
> > +
> > +/* For khugepaged_enter_vma(). */
> > +#include <linux/khugepaged.h>
> > +
> > +/* For maple tree operations. */
> > +#include <linux/maple_tree.h>
> > +
> > +/* For mpol_put(), vma_policy(), vma_dup_policy(), mpol_equal(). */
> > +#include <linux/mempolicy.h>
> > +
> > +/*
> > + * For VM flags, update_hiwater_rss(), __vm_area_free(), vm_area_free(),
> > + * vm_area_dup(), unmap_vmas(), vma_start_write(), vma_prev(), vma_next(),
> > + * vma_iter_free(), vm_area_free(), vm_stat_account(), vma_is_shared_maywrite(),
> > + * vma_interval_tree_remove(), anon_vma_interval_tree_remove(),
> > + * anon_vma_interval_tree_insert(), vma_assert_write_locked(), for_each_vma(),
> > + * vma_mark_detached(), for_each_vma_range(), vma_iter_set(),
> > + * vma_iter_prev_range(), vma_iter_clear_gfp(), PAGE_ALIGN(), vma_find(),
> > + * find_vma_intersection(), vma_lookup(), vma_is_anonymous(), find_vma_prev().
> > + */
> > +#include <linux/mm.h>
> > +
> > +/* For VM_WARN_ON(), VM_WARN_ON_ONCE_MM(), VM_BUG_ON_VMA(). */
> > +#include <linux/mmdebug.h>
> > +
> > +/* For list_is_singular(), list_for_each_entry(). */
> > +#include <linux/list.h>
> > +
> > +/* For anon_vma_name_eq(). */
> > +#include <linux/mm_inline.h>
> > +
> > +/* For vm_unacct_memory(), vma_pages(). */
> > +#include <linux/mman.h>
> > +
> > +/* For mmap_write_unlock(), mmap_write_downgrade(), mmap_assert_write_locked(). */
> > +#include <linux/mmap_lock.h>
> > +
> > +/* For DEFINE_MUTEX(), mutex_lock(), mutex_unlock(). */
> > +#include <linux/mutex.h>
> > +
> > +/* For AS_MM_ALL_LOCKS. */
> > +#include <linux/pagemap.h>
> > +
> > +/* For PHYS_PFN(). */
> > +#include <linux/pfn.h>
> > +
> > +/* For rcu_read_lock(), rcu_read_unlock(). */
> > +#include <linux/rcupdate.h>
> > +
> > +/*
> > + * For anon_vma_clone(), anon_vma_lock_write(), anon_vma_clone(),
> > + * unlink_anon_vmas(), anon_vma_unlock_write(), anon_vma_merge().
> > + */
> > +#include <linux/rmap.h>
> > +
> > +/* For down_write_nest_lock(). */
> > +#include <linux/rwsem.h>
> > +
> > +/* For signal_pending(). */
> > +#include <linux/sched/signal.h>
> > +
> > +/* For lru_add_drain(). */
> > +#include <linux/swap.h>
> > +
> > +/* For uprobe_mmap(), uprobe_munmap(). */
> > +#include <linux/uprobes.h>
> > +/*
> > + * For struct vm_userfaultfd_ctx, is_mergeable_vm_userfaultfd_ctx(),
> > + * userfaultfd_unmap_prep, userfaultfd_wp().
> > + */
> > +#include <linux/userfaultfd_k.h>
> > +
> > +/* For BUG(). */
> > +#include <linux/bug.h>
> > +
> > +/* For flush_dcache_mmap_lock(), flush_dcache_mmap_unlock(). */
> > +#include <linux/cacheflush.h>
> > +
> > +/* For current. */
> > +#include <asm/current.h>
> > +
> > +/* For PAGE_SHIFT, etc. */
> > +#include <asm/page_types.h>
> > +
> > +/* For pgprot_val(). */
> > +#include <asm/pgtable_types.h>
> > +
> > +/* For struct mmu_gather. */
> > +#include <asm/tlb.h>
> > +
> > +/* For arch_unmap(). */
> > +#include <linux/mmu_context.h>
> > +
> > +/*
> > + * For free_pgtables(), vma_set_range(), can_modify_mm(),
> > + * vma_soft_dirty_enabled().
> > + */
> > +#include "internal.h"
> > --
> > 2.45.1
> >




[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Bugtraq]     [Linux OMAP]     [Linux MIPS]     [eCos]     [Asterisk Internet PBX]     [Linux API]

  Powered by Linux