Re: [RFC PATCH 4/7] mm: move internal core VMA manipulation functions to own file

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



* Lorenzo Stoakes <lstoakes@xxxxxxxxx> [240627 06:39]:
> This patch introduces vma.c and moves internal core VMA manipulation
> functions to this file from mmap.c.
> 
> This allows us to isolate VMA functionality in a single place such that we
> can create userspace testing code that invokes this functionality in an
> environment where we can implement simple unit tests of core functionality.
> 
> This patch ensures that core VMA functionality is explicitly marked as such
> by its presence in mm/vma.h.
> 
> It also places the header includes required by vma.c in vma_internal.h,
> which is simply imported by vma.c. This makes the VMA functionality
> testable, as userland testing code can simply stub out functionality
> as required.

My initial thought on vma_internal.h would be to contain the number of
'helper' functions and internal structures while mm/vma.h would have the
interface.

In this way, we could include mm/vma.h into mm/internal.h (which most
files you've edited already has included), and any special cases
(mmu_notifier.c, etc) would need the addition.  vma_internal.h would
have only things needed in the vma.c file.

On testing, we could use the header guards to exclude what we wanted by
either just #defining the right guard, or by making an entirely new
header with a duplicate guard with the necessary stubs/functions.

> 
> Signed-off-by: Lorenzo Stoakes <lstoakes@xxxxxxxxx>
> ---
>  include/linux/mm.h |   35 -
>  mm/Makefile        |    2 +-
>  mm/gup.c           |    1 +
>  mm/huge_memory.c   |    1 +
>  mm/internal.h      |  227 +----
>  mm/madvise.c       |    1 +
>  mm/memory.c        |    1 +
>  mm/mempolicy.c     |    1 +
>  mm/mlock.c         |    1 +
>  mm/mmap.c          | 1983 +++-----------------------------------------
>  mm/mmu_notifier.c  |    2 +
>  mm/mprotect.c      |    1 +
>  mm/mremap.c        |    1 +
>  mm/mseal.c         |    2 +
>  mm/rmap.c          |    1 +
>  mm/userfaultfd.c   |    2 +
>  mm/vma.c           | 1766 +++++++++++++++++++++++++++++++++++++++
>  mm/vma.h           |  356 ++++++++
>  mm/vma_internal.h  |  143 ++++
>  19 files changed, 2389 insertions(+), 2138 deletions(-)
>  create mode 100644 mm/vma.c
>  create mode 100644 mm/vma.h
>  create mode 100644 mm/vma_internal.h
> 
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index e3220439cf75..31f85db029b8 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1004,21 +1004,6 @@ struct vm_area_struct *vma_iter_prev_range(struct vma_iterator *vmi)
>  	return mas_prev_range(&vmi->mas, 0);
>  }
>  
> -static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
> -{
> -	return vmi->mas.index;
> -}
> -
> -static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
> -{
> -	return vmi->mas.last + 1;
> -}
> -static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
> -				      unsigned long count)
> -{
> -	return mas_expected_entries(&vmi->mas, count);
> -}
> -
>  static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
>  			unsigned long start, unsigned long end, gfp_t gfp)
>  {
> @@ -2548,21 +2533,6 @@ extern unsigned long move_page_tables(struct vm_area_struct *vma,
>  #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
>  					    MM_CP_UFFD_WP_RESOLVE)
>  
> -bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
> -bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
> -static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
> -{
> -	/*
> -	 * We want to check manually if we can change individual PTEs writable
> -	 * if we can't do that automatically for all PTEs in a mapping. For
> -	 * private mappings, that's always the case when we have write
> -	 * permissions as we properly have to handle COW.
> -	 */
> -	if (vma->vm_flags & VM_SHARED)
> -		return vma_wants_writenotify(vma, vma->vm_page_prot);
> -	return !!(vma->vm_flags & VM_WRITE);
> -
> -}
>  bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
>  			     pte_t pte);
>  extern long change_protection(struct mmu_gather *tlb,
> @@ -3277,12 +3247,7 @@ extern int vma_expand_bottom(struct vma_iterator *vmi, struct vm_area_struct *vm
>  			     unsigned long shift, struct vm_area_struct **next);
>  extern int vma_shrink_top(struct vma_iterator *vmi, struct vm_area_struct *vma,
>  			  unsigned long shift);
> -extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
>  extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
> -extern void unlink_file_vma(struct vm_area_struct *);
> -extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
> -	unsigned long addr, unsigned long len, pgoff_t pgoff,
> -	bool *need_rmap_locks);
>  extern void exit_mmap(struct mm_struct *);
>  
>  static inline int check_data_rlimit(unsigned long rlim,
> diff --git a/mm/Makefile b/mm/Makefile
> index d2915f8c9dc0..140a22654dde 100644
> --- a/mm/Makefile
> +++ b/mm/Makefile
> @@ -37,7 +37,7 @@ mmu-y			:= nommu.o
>  mmu-$(CONFIG_MMU)	:= highmem.o memory.o mincore.o \
>  			   mlock.o mmap.o mmu_gather.o mprotect.o mremap.o \
>  			   msync.o page_vma_mapped.o pagewalk.o \
> -			   pgtable-generic.o rmap.o vmalloc.o
> +			   pgtable-generic.o rmap.o vmalloc.o vma.o
>  
>  
>  ifdef CONFIG_CROSS_MEMORY_ATTACH
> diff --git a/mm/gup.c b/mm/gup.c
> index 8bea9ad80984..34b846352679 100644
> --- a/mm/gup.c
> +++ b/mm/gup.c
> @@ -26,6 +26,7 @@
>  #include <asm/tlbflush.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  struct follow_page_context {
>  	struct dev_pagemap *pgmap;
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index c7ce28f6b7f3..de6f150ed97b 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -44,6 +44,7 @@
>  #include <asm/tlb.h>
>  #include <asm/pgalloc.h>
>  #include "internal.h"
> +#include "vma.h"
>  #include "swap.h"
>  
>  #define CREATE_TRACE_POINTS
> diff --git a/mm/internal.h b/mm/internal.h
> index f7779727bb78..76b4821cd751 100644
> --- a/mm/internal.h
> +++ b/mm/internal.h
> @@ -8,7 +8,9 @@
>  #define __MM_INTERNAL_H
>  
>  #include <linux/fs.h>
> +#include <linux/khugepaged.h>
>  #include <linux/mm.h>
> +#include <linux/mm_inline.h>
>  #include <linux/pagemap.h>
>  #include <linux/rmap.h>
>  #include <linux/swap.h>
> @@ -778,37 +780,6 @@ static inline bool free_area_empty(struct free_area *area, int migratetype)
>  	return list_empty(&area->free_list[migratetype]);
>  }
>  
> -/*
> - * These three helpers classifies VMAs for virtual memory accounting.
> - */
> -
> -/*
> - * Executable code area - executable, not writable, not stack
> - */
> -static inline bool is_exec_mapping(vm_flags_t flags)
> -{
> -	return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
> -}
> -
> -/*
> - * Stack area (including shadow stacks)
> - *
> - * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
> - * do_mmap() forbids all other combinations.
> - */
> -static inline bool is_stack_mapping(vm_flags_t flags)
> -{
> -	return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
> -}
> -
> -/*
> - * Data area - private, writable, not stack
> - */
> -static inline bool is_data_mapping(vm_flags_t flags)
> -{
> -	return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
> -}
> -
>  /* mm/util.c */
>  struct anon_vma *folio_anon_vma(struct folio *folio);
>  
> @@ -1237,80 +1208,6 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr,
>  void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
>  	       pmd_t *pmd, bool write);
>  
> -/*
> - * mm/mmap.c
> - */
> -struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> -					struct vm_area_struct *vma,
> -					unsigned long delta);
> -
> -struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> -				  struct vm_area_struct *prev,
> -				  struct vm_area_struct *vma,
> -				  unsigned long start, unsigned long end,
> -				  unsigned long vm_flags,
> -				  struct mempolicy *policy,
> -				  struct vm_userfaultfd_ctx uffd_ctx,
> -				  struct anon_vma_name *anon_name);
> -
> -/* We are about to modify the VMA's flags. */
> -static inline struct vm_area_struct
> -*vma_modify_flags(struct vma_iterator *vmi,
> -		  struct vm_area_struct *prev,
> -		  struct vm_area_struct *vma,
> -		  unsigned long start, unsigned long end,
> -		  unsigned long new_flags)
> -{
> -	return vma_modify(vmi, prev, vma, start, end, new_flags,
> -			  vma_policy(vma), vma->vm_userfaultfd_ctx,
> -			  anon_vma_name(vma));
> -}
> -
> -/* We are about to modify the VMA's flags and/or anon_name. */
> -static inline struct vm_area_struct
> -*vma_modify_flags_name(struct vma_iterator *vmi,
> -		       struct vm_area_struct *prev,
> -		       struct vm_area_struct *vma,
> -		       unsigned long start,
> -		       unsigned long end,
> -		       unsigned long new_flags,
> -		       struct anon_vma_name *new_name)
> -{
> -	return vma_modify(vmi, prev, vma, start, end, new_flags,
> -			  vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
> -}
> -
> -/* We are about to modify the VMA's memory policy. */
> -static inline struct vm_area_struct
> -*vma_modify_policy(struct vma_iterator *vmi,
> -		   struct vm_area_struct *prev,
> -		   struct vm_area_struct *vma,
> -		   unsigned long start, unsigned long end,
> -		   struct mempolicy *new_pol)
> -{
> -	return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
> -			  new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> -}
> -
> -/* We are about to modify the VMA's flags and/or uffd context. */
> -static inline struct vm_area_struct
> -*vma_modify_flags_uffd(struct vma_iterator *vmi,
> -		       struct vm_area_struct *prev,
> -		       struct vm_area_struct *vma,
> -		       unsigned long start, unsigned long end,
> -		       unsigned long new_flags,
> -		       struct vm_userfaultfd_ctx new_ctx)
> -{
> -	return vma_modify(vmi, prev, vma, start, end, new_flags,
> -			  vma_policy(vma), new_ctx, anon_vma_name(vma));
> -}
> -
> -int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -	       unsigned long start, unsigned long end, pgoff_t pgoff,
> -		      struct vm_area_struct *next);
> -int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -	       unsigned long start, unsigned long end, pgoff_t pgoff);
> -
>  enum {
>  	/* mark page accessed */
>  	FOLL_TOUCH = 1 << 16,
> @@ -1437,117 +1334,6 @@ static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte
>  	return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);
>  }
>  
> -static inline void vma_iter_config(struct vma_iterator *vmi,
> -		unsigned long index, unsigned long last)
> -{
> -	__mas_set_range(&vmi->mas, index, last - 1);
> -}
> -
> -static inline void vma_iter_reset(struct vma_iterator *vmi)
> -{
> -	mas_reset(&vmi->mas);
> -}
> -
> -static inline
> -struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
> -{
> -	return mas_prev_range(&vmi->mas, min);
> -}
> -
> -static inline
> -struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
> -{
> -	return mas_next_range(&vmi->mas, max);
> -}
> -
> -static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
> -				       unsigned long max, unsigned long size)
> -{
> -	return mas_empty_area(&vmi->mas, min, max - 1, size);
> -}
> -
> -static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
> -					unsigned long max, unsigned long size)
> -{
> -	return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
> -}
> -
> -/*
> - * VMA Iterator functions shared between nommu and mmap
> - */
> -static inline int vma_iter_prealloc(struct vma_iterator *vmi,
> -		struct vm_area_struct *vma)
> -{
> -	return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
> -}
> -
> -static inline void vma_iter_clear(struct vma_iterator *vmi)
> -{
> -	mas_store_prealloc(&vmi->mas, NULL);
> -}
> -
> -static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
> -{
> -	return mas_walk(&vmi->mas);
> -}
> -
> -/* Store a VMA with preallocated memory */
> -static inline void vma_iter_store(struct vma_iterator *vmi,
> -				  struct vm_area_struct *vma)
> -{
> -
> -#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> -	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> -			vmi->mas.index > vma->vm_start)) {
> -		pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
> -			vmi->mas.index, vma->vm_start, vma->vm_start,
> -			vma->vm_end, vmi->mas.index, vmi->mas.last);
> -	}
> -	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> -			vmi->mas.last <  vma->vm_start)) {
> -		pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
> -		       vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
> -		       vmi->mas.index, vmi->mas.last);
> -	}
> -#endif
> -
> -	if (vmi->mas.status != ma_start &&
> -	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> -		vma_iter_invalidate(vmi);
> -
> -	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> -	mas_store_prealloc(&vmi->mas, vma);
> -}
> -
> -static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
> -			struct vm_area_struct *vma, gfp_t gfp)
> -{
> -	if (vmi->mas.status != ma_start &&
> -	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> -		vma_iter_invalidate(vmi);
> -
> -	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> -	mas_store_gfp(&vmi->mas, vma, gfp);
> -	if (unlikely(mas_is_err(&vmi->mas)))
> -		return -ENOMEM;
> -
> -	return 0;
> -}
> -
> -/*
> - * VMA lock generalization
> - */
> -struct vma_prepare {
> -	struct vm_area_struct *vma;
> -	struct vm_area_struct *adj_next;
> -	struct file *file;
> -	struct address_space *mapping;
> -	struct anon_vma *anon_vma;
> -	struct vm_area_struct *insert;
> -	struct vm_area_struct *remove;
> -	struct vm_area_struct *remove2;
> -};
> -
>  void __meminit __init_single_page(struct page *page, unsigned long pfn,
>  				unsigned long zone, int nid);
>  
> @@ -1636,13 +1422,4 @@ static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry,
>  void workingset_update_node(struct xa_node *node);
>  extern struct list_lru shadow_nodes;
>  
> -struct unlink_vma_file_batch {
> -	int count;
> -	struct vm_area_struct *vmas[8];
> -};
> -
> -void unlink_file_vma_batch_init(struct unlink_vma_file_batch *);
> -void unlink_file_vma_batch_add(struct unlink_vma_file_batch *, struct vm_area_struct *);
> -void unlink_file_vma_batch_final(struct unlink_vma_file_batch *);
> -
>  #endif	/* __MM_INTERNAL_H */
> diff --git a/mm/madvise.c b/mm/madvise.c
> index 96c026fe0c99..42f62a8efd71 100644
> --- a/mm/madvise.c
> +++ b/mm/madvise.c
> @@ -35,6 +35,7 @@
>  #include <asm/tlb.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  #include "swap.h"
>  
>  struct madvise_walk_private {
> diff --git a/mm/memory.c b/mm/memory.c
> index 0a769f34bbb2..a2ca7df9d2cf 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -90,6 +90,7 @@
>  
>  #include "pgalloc-track.h"
>  #include "internal.h"
> +#include "vma.h"
>  #include "swap.h"
>  
>  #if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
> diff --git a/mm/mempolicy.c b/mm/mempolicy.c
> index f73acb01ad45..3dad2b52f319 100644
> --- a/mm/mempolicy.c
> +++ b/mm/mempolicy.c
> @@ -115,6 +115,7 @@
>  #include <linux/uaccess.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  /* Internal flags */
>  #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
> diff --git a/mm/mlock.c b/mm/mlock.c
> index 52d6e401ad67..ac84378bb796 100644
> --- a/mm/mlock.c
> +++ b/mm/mlock.c
> @@ -27,6 +27,7 @@
>  #include <linux/secretmem.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  struct mlock_fbatch {
>  	local_lock_t lock;
> diff --git a/mm/mmap.c b/mm/mmap.c
> index 574e69a04ebe..b4f7c1ea3f0f 100644
> --- a/mm/mmap.c
> +++ b/mm/mmap.c
> @@ -57,6 +57,7 @@
>  #include <trace/events/mmap.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  #ifndef arch_mmap_check
>  #define arch_mmap_check(addr, len, flags)	(0)
> @@ -76,16 +77,6 @@ int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
>  static bool ignore_rlimit_data;
>  core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
>  
> -static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> -		struct vm_area_struct *vma, struct vm_area_struct *prev,
> -		struct vm_area_struct *next, unsigned long start,
> -		unsigned long end, unsigned long tree_end, bool mm_wr_locked);
> -
> -static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
> -{
> -	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
> -}
> -
>  /* Update vma->vm_page_prot to reflect vma->vm_flags. */
>  void vma_set_page_prot(struct vm_area_struct *vma)
>  {
> @@ -101,100 +92,6 @@ void vma_set_page_prot(struct vm_area_struct *vma)
>  	WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
>  }
>  
> -/*
> - * Requires inode->i_mapping->i_mmap_rwsem
> - */
> -static void __remove_shared_vm_struct(struct vm_area_struct *vma,
> -				      struct address_space *mapping)
> -{
> -	if (vma_is_shared_maywrite(vma))
> -		mapping_unmap_writable(mapping);
> -
> -	flush_dcache_mmap_lock(mapping);
> -	vma_interval_tree_remove(vma, &mapping->i_mmap);
> -	flush_dcache_mmap_unlock(mapping);
> -}
> -
> -/*
> - * Unlink a file-based vm structure from its interval tree, to hide
> - * vma from rmap and vmtruncate before freeing its page tables.
> - */
> -void unlink_file_vma(struct vm_area_struct *vma)
> -{
> -	struct file *file = vma->vm_file;
> -
> -	if (file) {
> -		struct address_space *mapping = file->f_mapping;
> -		i_mmap_lock_write(mapping);
> -		__remove_shared_vm_struct(vma, mapping);
> -		i_mmap_unlock_write(mapping);
> -	}
> -}
> -
> -void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)
> -{
> -	vb->count = 0;
> -}
> -
> -static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb)
> -{
> -	struct address_space *mapping;
> -	int i;
> -
> -	mapping = vb->vmas[0]->vm_file->f_mapping;
> -	i_mmap_lock_write(mapping);
> -	for (i = 0; i < vb->count; i++) {
> -		VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping);
> -		__remove_shared_vm_struct(vb->vmas[i], mapping);
> -	}
> -	i_mmap_unlock_write(mapping);
> -
> -	unlink_file_vma_batch_init(vb);
> -}
> -
> -void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
> -			       struct vm_area_struct *vma)
> -{
> -	if (vma->vm_file == NULL)
> -		return;
> -
> -	if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) ||
> -	    vb->count == ARRAY_SIZE(vb->vmas))
> -		unlink_file_vma_batch_process(vb);
> -
> -	vb->vmas[vb->count] = vma;
> -	vb->count++;
> -}
> -
> -void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb)
> -{
> -	if (vb->count > 0)
> -		unlink_file_vma_batch_process(vb);
> -}
> -
> -/*
> - * Close a vm structure and free it.
> - */
> -static void remove_vma(struct vm_area_struct *vma, bool unreachable)
> -{
> -	might_sleep();
> -	if (vma->vm_ops && vma->vm_ops->close)
> -		vma->vm_ops->close(vma);
> -	if (vma->vm_file)
> -		fput(vma->vm_file);
> -	mpol_put(vma_policy(vma));
> -	if (unreachable)
> -		__vm_area_free(vma);
> -	else
> -		vm_area_free(vma);
> -}
> -
> -static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
> -						    unsigned long min)
> -{
> -	return mas_prev(&vmi->mas, min);
> -}
> -
>  /*
>   * check_brk_limits() - Use platform specific check of range & verify mlock
>   * limits.
> @@ -298,893 +195,24 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
>  	brkvma = vma_prev_limit(&vmi, mm->start_brk);
>  	/* Ok, looks good - let it rip. */
>  	if (do_brk_flags(&vmi, brkvma, oldbrk, newbrk - oldbrk, 0) < 0)
> -		goto out;
> -
> -	mm->brk = brk;
> -	if (mm->def_flags & VM_LOCKED)
> -		populate = true;
> -
> -success:
> -	mmap_write_unlock(mm);
> -success_unlocked:
> -	userfaultfd_unmap_complete(mm, &uf);
> -	if (populate)
> -		mm_populate(oldbrk, newbrk - oldbrk);
> -	return brk;
> -
> -out:
> -	mm->brk = origbrk;
> -	mmap_write_unlock(mm);
> -	return origbrk;
> -}
> -
> -#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> -static void validate_mm(struct mm_struct *mm)
> -{
> -	int bug = 0;
> -	int i = 0;
> -	struct vm_area_struct *vma;
> -	VMA_ITERATOR(vmi, mm, 0);
> -
> -	mt_validate(&mm->mm_mt);
> -	for_each_vma(vmi, vma) {
> -#ifdef CONFIG_DEBUG_VM_RB
> -		struct anon_vma *anon_vma = vma->anon_vma;
> -		struct anon_vma_chain *avc;
> -#endif
> -		unsigned long vmi_start, vmi_end;
> -		bool warn = 0;
> -
> -		vmi_start = vma_iter_addr(&vmi);
> -		vmi_end = vma_iter_end(&vmi);
> -		if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
> -			warn = 1;
> -
> -		if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
> -			warn = 1;
> -
> -		if (warn) {
> -			pr_emerg("issue in %s\n", current->comm);
> -			dump_stack();
> -			dump_vma(vma);
> -			pr_emerg("tree range: %px start %lx end %lx\n", vma,
> -				 vmi_start, vmi_end - 1);
> -			vma_iter_dump_tree(&vmi);
> -		}
> -
> -#ifdef CONFIG_DEBUG_VM_RB
> -		if (anon_vma) {
> -			anon_vma_lock_read(anon_vma);
> -			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> -				anon_vma_interval_tree_verify(avc);
> -			anon_vma_unlock_read(anon_vma);
> -		}
> -#endif
> -		i++;
> -	}
> -	if (i != mm->map_count) {
> -		pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
> -		bug = 1;
> -	}
> -	VM_BUG_ON_MM(bug, mm);
> -}
> -
> -#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */
> -#define validate_mm(mm) do { } while (0)
> -#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
> -
> -/*
> - * vma has some anon_vma assigned, and is already inserted on that
> - * anon_vma's interval trees.
> - *
> - * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
> - * vma must be removed from the anon_vma's interval trees using
> - * anon_vma_interval_tree_pre_update_vma().
> - *
> - * After the update, the vma will be reinserted using
> - * anon_vma_interval_tree_post_update_vma().
> - *
> - * The entire update must be protected by exclusive mmap_lock and by
> - * the root anon_vma's mutex.
> - */
> -static inline void
> -anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
> -{
> -	struct anon_vma_chain *avc;
> -
> -	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> -		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
> -}
> -
> -static inline void
> -anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
> -{
> -	struct anon_vma_chain *avc;
> -
> -	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> -		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
> -}
> -
> -static unsigned long count_vma_pages_range(struct mm_struct *mm,
> -		unsigned long addr, unsigned long end)
> -{
> -	VMA_ITERATOR(vmi, mm, addr);
> -	struct vm_area_struct *vma;
> -	unsigned long nr_pages = 0;
> -
> -	for_each_vma_range(vmi, vma, end) {
> -		unsigned long vm_start = max(addr, vma->vm_start);
> -		unsigned long vm_end = min(end, vma->vm_end);
> -
> -		nr_pages += PHYS_PFN(vm_end - vm_start);
> -	}
> -
> -	return nr_pages;
> -}
> -
> -static void __vma_link_file(struct vm_area_struct *vma,
> -			    struct address_space *mapping)
> -{
> -	if (vma_is_shared_maywrite(vma))
> -		mapping_allow_writable(mapping);
> -
> -	flush_dcache_mmap_lock(mapping);
> -	vma_interval_tree_insert(vma, &mapping->i_mmap);
> -	flush_dcache_mmap_unlock(mapping);
> -}
> -
> -static void vma_link_file(struct vm_area_struct *vma)
> -{
> -	struct file *file = vma->vm_file;
> -	struct address_space *mapping;
> -
> -	if (file) {
> -		mapping = file->f_mapping;
> -		i_mmap_lock_write(mapping);
> -		__vma_link_file(vma, mapping);
> -		i_mmap_unlock_write(mapping);
> -	}
> -}
> -
> -static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
> -{
> -	VMA_ITERATOR(vmi, mm, 0);
> -
> -	vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
> -	if (vma_iter_prealloc(&vmi, vma))
> -		return -ENOMEM;
> -
> -	vma_start_write(vma);
> -	vma_iter_store(&vmi, vma);
> -	vma_link_file(vma);
> -	mm->map_count++;
> -	validate_mm(mm);
> -	return 0;
> -}
> -
> -/*
> - * init_multi_vma_prep() - Initializer for struct vma_prepare
> - * @vp: The vma_prepare struct
> - * @vma: The vma that will be altered once locked
> - * @next: The next vma if it is to be adjusted
> - * @remove: The first vma to be removed
> - * @remove2: The second vma to be removed
> - */
> -static inline void init_multi_vma_prep(struct vma_prepare *vp,
> -		struct vm_area_struct *vma, struct vm_area_struct *next,
> -		struct vm_area_struct *remove, struct vm_area_struct *remove2)
> -{
> -	memset(vp, 0, sizeof(struct vma_prepare));
> -	vp->vma = vma;
> -	vp->anon_vma = vma->anon_vma;
> -	vp->remove = remove;
> -	vp->remove2 = remove2;
> -	vp->adj_next = next;
> -	if (!vp->anon_vma && next)
> -		vp->anon_vma = next->anon_vma;
> -
> -	vp->file = vma->vm_file;
> -	if (vp->file)
> -		vp->mapping = vma->vm_file->f_mapping;
> -
> -}
> -
> -/*
> - * init_vma_prep() - Initializer wrapper for vma_prepare struct
> - * @vp: The vma_prepare struct
> - * @vma: The vma that will be altered once locked
> - */
> -static inline void init_vma_prep(struct vma_prepare *vp,
> -				 struct vm_area_struct *vma)
> -{
> -	init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
> -}
> -
> -
> -/*
> - * vma_prepare() - Helper function for handling locking VMAs prior to altering
> - * @vp: The initialized vma_prepare struct
> - */
> -static inline void vma_prepare(struct vma_prepare *vp)
> -{
> -	if (vp->file) {
> -		uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
> -
> -		if (vp->adj_next)
> -			uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
> -				      vp->adj_next->vm_end);
> -
> -		i_mmap_lock_write(vp->mapping);
> -		if (vp->insert && vp->insert->vm_file) {
> -			/*
> -			 * Put into interval tree now, so instantiated pages
> -			 * are visible to arm/parisc __flush_dcache_page
> -			 * throughout; but we cannot insert into address
> -			 * space until vma start or end is updated.
> -			 */
> -			__vma_link_file(vp->insert,
> -					vp->insert->vm_file->f_mapping);
> -		}
> -	}
> -
> -	if (vp->anon_vma) {
> -		anon_vma_lock_write(vp->anon_vma);
> -		anon_vma_interval_tree_pre_update_vma(vp->vma);
> -		if (vp->adj_next)
> -			anon_vma_interval_tree_pre_update_vma(vp->adj_next);
> -	}
> -
> -	if (vp->file) {
> -		flush_dcache_mmap_lock(vp->mapping);
> -		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
> -		if (vp->adj_next)
> -			vma_interval_tree_remove(vp->adj_next,
> -						 &vp->mapping->i_mmap);
> -	}
> -
> -}
> -
> -/*
> - * vma_complete- Helper function for handling the unlocking after altering VMAs,
> - * or for inserting a VMA.
> - *
> - * @vp: The vma_prepare struct
> - * @vmi: The vma iterator
> - * @mm: The mm_struct
> - */
> -static inline void vma_complete(struct vma_prepare *vp,
> -				struct vma_iterator *vmi, struct mm_struct *mm)
> -{
> -	if (vp->file) {
> -		if (vp->adj_next)
> -			vma_interval_tree_insert(vp->adj_next,
> -						 &vp->mapping->i_mmap);
> -		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
> -		flush_dcache_mmap_unlock(vp->mapping);
> -	}
> -
> -	if (vp->remove && vp->file) {
> -		__remove_shared_vm_struct(vp->remove, vp->mapping);
> -		if (vp->remove2)
> -			__remove_shared_vm_struct(vp->remove2, vp->mapping);
> -	} else if (vp->insert) {
> -		/*
> -		 * split_vma has split insert from vma, and needs
> -		 * us to insert it before dropping the locks
> -		 * (it may either follow vma or precede it).
> -		 */
> -		vma_iter_store(vmi, vp->insert);
> -		mm->map_count++;
> -	}
> -
> -	if (vp->anon_vma) {
> -		anon_vma_interval_tree_post_update_vma(vp->vma);
> -		if (vp->adj_next)
> -			anon_vma_interval_tree_post_update_vma(vp->adj_next);
> -		anon_vma_unlock_write(vp->anon_vma);
> -	}
> -
> -	if (vp->file) {
> -		i_mmap_unlock_write(vp->mapping);
> -		uprobe_mmap(vp->vma);
> -
> -		if (vp->adj_next)
> -			uprobe_mmap(vp->adj_next);
> -	}
> -
> -	if (vp->remove) {
> -again:
> -		vma_mark_detached(vp->remove, true);
> -		if (vp->file) {
> -			uprobe_munmap(vp->remove, vp->remove->vm_start,
> -				      vp->remove->vm_end);
> -			fput(vp->file);
> -		}
> -		if (vp->remove->anon_vma)
> -			anon_vma_merge(vp->vma, vp->remove);
> -		mm->map_count--;
> -		mpol_put(vma_policy(vp->remove));
> -		if (!vp->remove2)
> -			WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
> -		vm_area_free(vp->remove);
> -
> -		/*
> -		 * In mprotect's case 6 (see comments on vma_merge),
> -		 * we are removing both mid and next vmas
> -		 */
> -		if (vp->remove2) {
> -			vp->remove = vp->remove2;
> -			vp->remove2 = NULL;
> -			goto again;
> -		}
> -	}
> -	if (vp->insert && vp->file)
> -		uprobe_mmap(vp->insert);
> -	validate_mm(mm);
> -}
> -
> -/*
> - * dup_anon_vma() - Helper function to duplicate anon_vma
> - * @dst: The destination VMA
> - * @src: The source VMA
> - * @dup: Pointer to the destination VMA when successful.
> - *
> - * Returns: 0 on success.
> - */
> -static inline int dup_anon_vma(struct vm_area_struct *dst,
> -		struct vm_area_struct *src, struct vm_area_struct **dup)
> -{
> -	/*
> -	 * Easily overlooked: when mprotect shifts the boundary, make sure the
> -	 * expanding vma has anon_vma set if the shrinking vma had, to cover any
> -	 * anon pages imported.
> -	 */
> -	if (src->anon_vma && !dst->anon_vma) {
> -		int ret;
> -
> -		vma_assert_write_locked(dst);
> -		dst->anon_vma = src->anon_vma;
> -		ret = anon_vma_clone(dst, src);
> -		if (ret)
> -			return ret;
> -
> -		*dup = dst;
> -	}
> -
> -	return 0;
> -}
> -
> -/*
> - * vma_expand - Expand an existing VMA
> - *
> - * @vmi: The vma iterator
> - * @vma: The vma to expand
> - * @start: The start of the vma
> - * @end: The exclusive end of the vma
> - * @pgoff: The page offset of vma
> - * @next: The current of next vma.
> - *
> - * Expand @vma to @start and @end.  Can expand off the start and end.  Will
> - * expand over @next if it's different from @vma and @end == @next->vm_end.
> - * Checking if the @vma can expand and merge with @next needs to be handled by
> - * the caller.
> - *
> - * Returns: 0 on success
> - */
> -int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -	       unsigned long start, unsigned long end, pgoff_t pgoff,
> -	       struct vm_area_struct *next)
> -{
> -	struct vm_area_struct *anon_dup = NULL;
> -	bool remove_next = false;
> -	struct vma_prepare vp;
> -
> -	vma_start_write(vma);
> -	if (next && (vma != next) && (end == next->vm_end)) {
> -		int ret;
> -
> -		remove_next = true;
> -		vma_start_write(next);
> -		ret = dup_anon_vma(vma, next, &anon_dup);
> -		if (ret)
> -			return ret;
> -	}
> -
> -	init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL);
> -	/* Not merging but overwriting any part of next is not handled. */
> -	VM_WARN_ON(next && !vp.remove &&
> -		  next != vma && end > next->vm_start);
> -	/* Only handles expanding */
> -	VM_WARN_ON(vma->vm_start < start || vma->vm_end > end);
> -
> -	/* Note: vma iterator must be pointing to 'start' */
> -	vma_iter_config(vmi, start, end);
> -	if (vma_iter_prealloc(vmi, vma))
> -		goto nomem;
> -
> -	vma_prepare(&vp);
> -	vma_adjust_trans_huge(vma, start, end, 0);
> -	vma_set_range(vma, start, end, pgoff);
> -	vma_iter_store(vmi, vma);
> -
> -	vma_complete(&vp, vmi, vma->vm_mm);
> -	return 0;
> -
> -nomem:
> -	if (anon_dup)
> -		unlink_anon_vmas(anon_dup);
> -	return -ENOMEM;
> -}
> -
> -/*
> - * vma_shrink() - Reduce an existing VMAs memory area
> - * @vmi: The vma iterator
> - * @vma: The VMA to modify
> - * @start: The new start
> - * @end: The new end
> - *
> - * Returns: 0 on success, -ENOMEM otherwise
> - */
> -int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -	       unsigned long start, unsigned long end, pgoff_t pgoff)
> -{
> -	struct vma_prepare vp;
> -
> -	WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
> -
> -	if (vma->vm_start < start)
> -		vma_iter_config(vmi, vma->vm_start, start);
> -	else
> -		vma_iter_config(vmi, end, vma->vm_end);
> -
> -	if (vma_iter_prealloc(vmi, NULL))
> -		return -ENOMEM;
> -
> -	vma_start_write(vma);
> -
> -	init_vma_prep(&vp, vma);
> -	vma_prepare(&vp);
> -	vma_adjust_trans_huge(vma, start, end, 0);
> -
> -	vma_iter_clear(vmi);
> -	vma_set_range(vma, start, end, pgoff);
> -	vma_complete(&vp, vmi, vma->vm_mm);
> -	return 0;
> -}
> -
> -/*
> - * If the vma has a ->close operation then the driver probably needs to release
> - * per-vma resources, so we don't attempt to merge those if the caller indicates
> - * the current vma may be removed as part of the merge.
> - */
> -static inline bool is_mergeable_vma(struct vm_area_struct *vma,
> -		struct file *file, unsigned long vm_flags,
> -		struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> -		struct anon_vma_name *anon_name, bool may_remove_vma)
> -{
> -	/*
> -	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
> -	 * match the flags but dirty bit -- the caller should mark
> -	 * merged VMA as dirty. If dirty bit won't be excluded from
> -	 * comparison, we increase pressure on the memory system forcing
> -	 * the kernel to generate new VMAs when old one could be
> -	 * extended instead.
> -	 */
> -	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
> -		return false;
> -	if (vma->vm_file != file)
> -		return false;
> -	if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
> -		return false;
> -	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
> -		return false;
> -	if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
> -		return false;
> -	return true;
> -}
> -
> -static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
> -		 struct anon_vma *anon_vma2, struct vm_area_struct *vma)
> -{
> -	/*
> -	 * The list_is_singular() test is to avoid merging VMA cloned from
> -	 * parents. This can improve scalability caused by anon_vma lock.
> -	 */
> -	if ((!anon_vma1 || !anon_vma2) && (!vma ||
> -		list_is_singular(&vma->anon_vma_chain)))
> -		return true;
> -	return anon_vma1 == anon_vma2;
> -}
> -
> -/*
> - * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> - * in front of (at a lower virtual address and file offset than) the vma.
> - *
> - * We cannot merge two vmas if they have differently assigned (non-NULL)
> - * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> - *
> - * We don't check here for the merged mmap wrapping around the end of pagecache
> - * indices (16TB on ia32) because do_mmap() does not permit mmap's which
> - * wrap, nor mmaps which cover the final page at index -1UL.
> - *
> - * We assume the vma may be removed as part of the merge.
> - */
> -static bool
> -can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
> -		struct anon_vma *anon_vma, struct file *file,
> -		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> -		struct anon_vma_name *anon_name)
> -{
> -	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&
> -	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> -		if (vma->vm_pgoff == vm_pgoff)
> -			return true;
> -	}
> -	return false;
> -}
> -
> -/*
> - * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> - * beyond (at a higher virtual address and file offset than) the vma.
> - *
> - * We cannot merge two vmas if they have differently assigned (non-NULL)
> - * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> - *
> - * We assume that vma is not removed as part of the merge.
> - */
> -static bool
> -can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
> -		struct anon_vma *anon_vma, struct file *file,
> -		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> -		struct anon_vma_name *anon_name)
> -{
> -	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&
> -	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> -		pgoff_t vm_pglen;
> -		vm_pglen = vma_pages(vma);
> -		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
> -			return true;
> -	}
> -	return false;
> -}
> -
> -/*
> - * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
> - * figure out whether that can be merged with its predecessor or its
> - * successor.  Or both (it neatly fills a hole).
> - *
> - * In most cases - when called for mmap, brk or mremap - [addr,end) is
> - * certain not to be mapped by the time vma_merge is called; but when
> - * called for mprotect, it is certain to be already mapped (either at
> - * an offset within prev, or at the start of next), and the flags of
> - * this area are about to be changed to vm_flags - and the no-change
> - * case has already been eliminated.
> - *
> - * The following mprotect cases have to be considered, where **** is
> - * the area passed down from mprotect_fixup, never extending beyond one
> - * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts
> - * at the same address as **** and is of the same or larger span, and
> - * NNNN the next vma after ****:
> - *
> - *     ****             ****                   ****
> - *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPCCCCCC
> - *    cannot merge    might become       might become
> - *                    PPNNNNNNNNNN       PPPPPPPPPPCC
> - *    mmap, brk or    case 4 below       case 5 below
> - *    mremap move:
> - *                        ****               ****
> - *                    PPPP    NNNN       PPPPCCCCNNNN
> - *                    might become       might become
> - *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
> - *                    PPPPPPPPNNNN 2 or  PPPPPPPPNNNN 7 or
> - *                    PPPPNNNNNNNN 3     PPPPNNNNNNNN 8
> - *
> - * It is important for case 8 that the vma CCCC overlapping the
> - * region **** is never going to extended over NNNN. Instead NNNN must
> - * be extended in region **** and CCCC must be removed. This way in
> - * all cases where vma_merge succeeds, the moment vma_merge drops the
> - * rmap_locks, the properties of the merged vma will be already
> - * correct for the whole merged range. Some of those properties like
> - * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
> - * be correct for the whole merged range immediately after the
> - * rmap_locks are released. Otherwise if NNNN would be removed and
> - * CCCC would be extended over the NNNN range, remove_migration_ptes
> - * or other rmap walkers (if working on addresses beyond the "end"
> - * parameter) may establish ptes with the wrong permissions of CCCC
> - * instead of the right permissions of NNNN.
> - *
> - * In the code below:
> - * PPPP is represented by *prev
> - * CCCC is represented by *curr or not represented at all (NULL)
> - * NNNN is represented by *next or not represented at all (NULL)
> - * **** is not represented - it will be merged and the vma containing the
> - *      area is returned, or the function will return NULL
> - */
> -static struct vm_area_struct
> -*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
> -	   struct vm_area_struct *src, unsigned long addr, unsigned long end,
> -	   unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
> -	   struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> -	   struct anon_vma_name *anon_name)
> -{
> -	struct mm_struct *mm = src->vm_mm;
> -	struct anon_vma *anon_vma = src->anon_vma;
> -	struct file *file = src->vm_file;
> -	struct vm_area_struct *curr, *next, *res;
> -	struct vm_area_struct *vma, *adjust, *remove, *remove2;
> -	struct vm_area_struct *anon_dup = NULL;
> -	struct vma_prepare vp;
> -	pgoff_t vma_pgoff;
> -	int err = 0;
> -	bool merge_prev = false;
> -	bool merge_next = false;
> -	bool vma_expanded = false;
> -	unsigned long vma_start = addr;
> -	unsigned long vma_end = end;
> -	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
> -	long adj_start = 0;
> -
> -	/*
> -	 * We later require that vma->vm_flags == vm_flags,
> -	 * so this tests vma->vm_flags & VM_SPECIAL, too.
> -	 */
> -	if (vm_flags & VM_SPECIAL)
> -		return NULL;
> -
> -	/* Does the input range span an existing VMA? (cases 5 - 8) */
> -	curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end);
> -
> -	if (!curr ||			/* cases 1 - 4 */
> -	    end == curr->vm_end)	/* cases 6 - 8, adjacent VMA */
> -		next = vma_lookup(mm, end);
> -	else
> -		next = NULL;		/* case 5 */
> -
> -	if (prev) {
> -		vma_start = prev->vm_start;
> -		vma_pgoff = prev->vm_pgoff;
> -
> -		/* Can we merge the predecessor? */
> -		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)
> -		    && can_vma_merge_after(prev, vm_flags, anon_vma, file,
> -					   pgoff, vm_userfaultfd_ctx, anon_name)) {
> -			merge_prev = true;
> -			vma_prev(vmi);
> -		}
> -	}
> -
> -	/* Can we merge the successor? */
> -	if (next && mpol_equal(policy, vma_policy(next)) &&
> -	    can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,
> -				 vm_userfaultfd_ctx, anon_name)) {
> -		merge_next = true;
> -	}
> -
> -	/* Verify some invariant that must be enforced by the caller. */
> -	VM_WARN_ON(prev && addr <= prev->vm_start);
> -	VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end));
> -	VM_WARN_ON(addr >= end);
> -
> -	if (!merge_prev && !merge_next)
> -		return NULL; /* Not mergeable. */
> -
> -	if (merge_prev)
> -		vma_start_write(prev);
> -
> -	res = vma = prev;
> -	remove = remove2 = adjust = NULL;
> -
> -	/* Can we merge both the predecessor and the successor? */
> -	if (merge_prev && merge_next &&
> -	    is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) {
> -		vma_start_write(next);
> -		remove = next;				/* case 1 */
> -		vma_end = next->vm_end;
> -		err = dup_anon_vma(prev, next, &anon_dup);
> -		if (curr) {				/* case 6 */
> -			vma_start_write(curr);
> -			remove = curr;
> -			remove2 = next;
> -			/*
> -			 * Note that the dup_anon_vma below cannot overwrite err
> -			 * since the first caller would do nothing unless next
> -			 * has an anon_vma.
> -			 */
> -			if (!next->anon_vma)
> -				err = dup_anon_vma(prev, curr, &anon_dup);
> -		}
> -	} else if (merge_prev) {			/* case 2 */
> -		if (curr) {
> -			vma_start_write(curr);
> -			if (end == curr->vm_end) {	/* case 7 */
> -				/*
> -				 * can_vma_merge_after() assumed we would not be
> -				 * removing prev vma, so it skipped the check
> -				 * for vm_ops->close, but we are removing curr
> -				 */
> -				if (curr->vm_ops && curr->vm_ops->close)
> -					err = -EINVAL;
> -				remove = curr;
> -			} else {			/* case 5 */
> -				adjust = curr;
> -				adj_start = (end - curr->vm_start);
> -			}
> -			if (!err)
> -				err = dup_anon_vma(prev, curr, &anon_dup);
> -		}
> -	} else { /* merge_next */
> -		vma_start_write(next);
> -		res = next;
> -		if (prev && addr < prev->vm_end) {	/* case 4 */
> -			vma_start_write(prev);
> -			vma_end = addr;
> -			adjust = next;
> -			adj_start = -(prev->vm_end - addr);
> -			err = dup_anon_vma(next, prev, &anon_dup);
> -		} else {
> -			/*
> -			 * Note that cases 3 and 8 are the ONLY ones where prev
> -			 * is permitted to be (but is not necessarily) NULL.
> -			 */
> -			vma = next;			/* case 3 */
> -			vma_start = addr;
> -			vma_end = next->vm_end;
> -			vma_pgoff = next->vm_pgoff - pglen;
> -			if (curr) {			/* case 8 */
> -				vma_pgoff = curr->vm_pgoff;
> -				vma_start_write(curr);
> -				remove = curr;
> -				err = dup_anon_vma(next, curr, &anon_dup);
> -			}
> -		}
> -	}
> -
> -	/* Error in anon_vma clone. */
> -	if (err)
> -		goto anon_vma_fail;
> -
> -	if (vma_start < vma->vm_start || vma_end > vma->vm_end)
> -		vma_expanded = true;
> -
> -	if (vma_expanded) {
> -		vma_iter_config(vmi, vma_start, vma_end);
> -	} else {
> -		vma_iter_config(vmi, adjust->vm_start + adj_start,
> -				adjust->vm_end);
> -	}
> -
> -	if (vma_iter_prealloc(vmi, vma))
> -		goto prealloc_fail;
> -
> -	init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
> -	VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
> -		   vp.anon_vma != adjust->anon_vma);
> -
> -	vma_prepare(&vp);
> -	vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start);
> -	vma_set_range(vma, vma_start, vma_end, vma_pgoff);
> -
> -	if (vma_expanded)
> -		vma_iter_store(vmi, vma);
> -
> -	if (adj_start) {
> -		adjust->vm_start += adj_start;
> -		adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
> -		if (adj_start < 0) {
> -			WARN_ON(vma_expanded);
> -			vma_iter_store(vmi, next);
> -		}
> -	}
> -
> -	vma_complete(&vp, vmi, mm);
> -	khugepaged_enter_vma(res, vm_flags);
> -	return res;
> -
> -prealloc_fail:
> -	if (anon_dup)
> -		unlink_anon_vmas(anon_dup);
> -
> -anon_vma_fail:
> -	vma_iter_set(vmi, addr);
> -	vma_iter_load(vmi);
> -	return NULL;
> -}
> -
> -/*
> - * Rough compatibility check to quickly see if it's even worth looking
> - * at sharing an anon_vma.
> - *
> - * They need to have the same vm_file, and the flags can only differ
> - * in things that mprotect may change.
> - *
> - * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
> - * we can merge the two vma's. For example, we refuse to merge a vma if
> - * there is a vm_ops->close() function, because that indicates that the
> - * driver is doing some kind of reference counting. But that doesn't
> - * really matter for the anon_vma sharing case.
> - */
> -static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
> -{
> -	return a->vm_end == b->vm_start &&
> -		mpol_equal(vma_policy(a), vma_policy(b)) &&
> -		a->vm_file == b->vm_file &&
> -		!((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
> -		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
> -}
> -
> -/*
> - * Do some basic sanity checking to see if we can re-use the anon_vma
> - * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
> - * the same as 'old', the other will be the new one that is trying
> - * to share the anon_vma.
> - *
> - * NOTE! This runs with mmap_lock held for reading, so it is possible that
> - * the anon_vma of 'old' is concurrently in the process of being set up
> - * by another page fault trying to merge _that_. But that's ok: if it
> - * is being set up, that automatically means that it will be a singleton
> - * acceptable for merging, so we can do all of this optimistically. But
> - * we do that READ_ONCE() to make sure that we never re-load the pointer.
> - *
> - * IOW: that the "list_is_singular()" test on the anon_vma_chain only
> - * matters for the 'stable anon_vma' case (ie the thing we want to avoid
> - * is to return an anon_vma that is "complex" due to having gone through
> - * a fork).
> - *
> - * We also make sure that the two vma's are compatible (adjacent,
> - * and with the same memory policies). That's all stable, even with just
> - * a read lock on the mmap_lock.
> - */
> -static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
> -{
> -	if (anon_vma_compatible(a, b)) {
> -		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
> -
> -		if (anon_vma && list_is_singular(&old->anon_vma_chain))
> -			return anon_vma;
> -	}
> -	return NULL;
> -}
> +		goto out;
>  
> -/*
> - * find_mergeable_anon_vma is used by anon_vma_prepare, to check
> - * neighbouring vmas for a suitable anon_vma, before it goes off
> - * to allocate a new anon_vma.  It checks because a repetitive
> - * sequence of mprotects and faults may otherwise lead to distinct
> - * anon_vmas being allocated, preventing vma merge in subsequent
> - * mprotect.
> - */
> -struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
> -{
> -	struct anon_vma *anon_vma = NULL;
> -	struct vm_area_struct *prev, *next;
> -	VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end);
> -
> -	/* Try next first. */
> -	next = vma_iter_load(&vmi);
> -	if (next) {
> -		anon_vma = reusable_anon_vma(next, vma, next);
> -		if (anon_vma)
> -			return anon_vma;
> -	}
> +	mm->brk = brk;
> +	if (mm->def_flags & VM_LOCKED)
> +		populate = true;
>  
> -	prev = vma_prev(&vmi);
> -	VM_BUG_ON_VMA(prev != vma, vma);
> -	prev = vma_prev(&vmi);
> -	/* Try prev next. */
> -	if (prev)
> -		anon_vma = reusable_anon_vma(prev, prev, vma);
> +success:
> +	mmap_write_unlock(mm);
> +success_unlocked:
> +	userfaultfd_unmap_complete(mm, &uf);
> +	if (populate)
> +		mm_populate(oldbrk, newbrk - oldbrk);
> +	return brk;
>  
> -	/*
> -	 * We might reach here with anon_vma == NULL if we can't find
> -	 * any reusable anon_vma.
> -	 * There's no absolute need to look only at touching neighbours:
> -	 * we could search further afield for "compatible" anon_vmas.
> -	 * But it would probably just be a waste of time searching,
> -	 * or lead to too many vmas hanging off the same anon_vma.
> -	 * We're trying to allow mprotect remerging later on,
> -	 * not trying to minimize memory used for anon_vmas.
> -	 */
> -	return anon_vma;
> +out:
> +	mm->brk = origbrk;
> +	mmap_write_unlock(mm);
> +	return origbrk;
>  }
>  
>  /*
> @@ -1519,85 +547,6 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
>  }
>  #endif /* __ARCH_WANT_SYS_OLD_MMAP */
>  
> -static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
> -{
> -	return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
> -}
> -
> -static bool vma_is_shared_writable(struct vm_area_struct *vma)
> -{
> -	return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
> -		(VM_WRITE | VM_SHARED);
> -}
> -
> -static bool vma_fs_can_writeback(struct vm_area_struct *vma)
> -{
> -	/* No managed pages to writeback. */
> -	if (vma->vm_flags & VM_PFNMAP)
> -		return false;
> -
> -	return vma->vm_file && vma->vm_file->f_mapping &&
> -		mapping_can_writeback(vma->vm_file->f_mapping);
> -}
> -
> -/*
> - * Does this VMA require the underlying folios to have their dirty state
> - * tracked?
> - */
> -bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
> -{
> -	/* Only shared, writable VMAs require dirty tracking. */
> -	if (!vma_is_shared_writable(vma))
> -		return false;
> -
> -	/* Does the filesystem need to be notified? */
> -	if (vm_ops_needs_writenotify(vma->vm_ops))
> -		return true;
> -
> -	/*
> -	 * Even if the filesystem doesn't indicate a need for writenotify, if it
> -	 * can writeback, dirty tracking is still required.
> -	 */
> -	return vma_fs_can_writeback(vma);
> -}
> -
> -/*
> - * Some shared mappings will want the pages marked read-only
> - * to track write events. If so, we'll downgrade vm_page_prot
> - * to the private version (using protection_map[] without the
> - * VM_SHARED bit).
> - */
> -bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
> -{
> -	/* If it was private or non-writable, the write bit is already clear */
> -	if (!vma_is_shared_writable(vma))
> -		return false;
> -
> -	/* The backer wishes to know when pages are first written to? */
> -	if (vm_ops_needs_writenotify(vma->vm_ops))
> -		return true;
> -
> -	/* The open routine did something to the protections that pgprot_modify
> -	 * won't preserve? */
> -	if (pgprot_val(vm_page_prot) !=
> -	    pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
> -		return false;
> -
> -	/*
> -	 * Do we need to track softdirty? hugetlb does not support softdirty
> -	 * tracking yet.
> -	 */
> -	if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
> -		return true;
> -
> -	/* Do we need write faults for uffd-wp tracking? */
> -	if (userfaultfd_wp(vma))
> -		return true;
> -
> -	/* Can the mapping track the dirty pages? */
> -	return vma_fs_can_writeback(vma);
> -}
> -
>  /*
>   * We account for memory if it's a private writeable mapping,
>   * not hugepages and VM_NORESERVE wasn't set.
> @@ -2238,566 +1187,129 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address)
>  				anon_vma_interval_tree_post_update_vma(vma);
>  				spin_unlock(&mm->page_table_lock);
>  
> -				perf_event_mmap(vma);
> -			}
> -		}
> -	}
> -	anon_vma_unlock_write(vma->anon_vma);
> -	vma_iter_free(&vmi);
> -	validate_mm(mm);
> -	return error;
> -}
> -
> -/* enforced gap between the expanding stack and other mappings. */
> -unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
> -
> -static int __init cmdline_parse_stack_guard_gap(char *p)
> -{
> -	unsigned long val;
> -	char *endptr;
> -
> -	val = simple_strtoul(p, &endptr, 10);
> -	if (!*endptr)
> -		stack_guard_gap = val << PAGE_SHIFT;
> -
> -	return 1;
> -}
> -__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
> -
> -#ifdef CONFIG_STACK_GROWSUP
> -int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
> -{
> -	return expand_upwards(vma, address);
> -}
> -
> -struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> -{
> -	struct vm_area_struct *vma, *prev;
> -
> -	addr &= PAGE_MASK;
> -	vma = find_vma_prev(mm, addr, &prev);
> -	if (vma && (vma->vm_start <= addr))
> -		return vma;
> -	if (!prev)
> -		return NULL;
> -	if (expand_stack_locked(prev, addr))
> -		return NULL;
> -	if (prev->vm_flags & VM_LOCKED)
> -		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
> -	return prev;
> -}
> -#else
> -int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
> -{
> -	return expand_downwards(vma, address);
> -}
> -
> -struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> -{
> -	struct vm_area_struct *vma;
> -	unsigned long start;
> -
> -	addr &= PAGE_MASK;
> -	vma = find_vma(mm, addr);
> -	if (!vma)
> -		return NULL;
> -	if (vma->vm_start <= addr)
> -		return vma;
> -	start = vma->vm_start;
> -	if (expand_stack_locked(vma, addr))
> -		return NULL;
> -	if (vma->vm_flags & VM_LOCKED)
> -		populate_vma_page_range(vma, addr, start, NULL);
> -	return vma;
> -}
> -#endif
> -
> -#if defined(CONFIG_STACK_GROWSUP)
> -
> -#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
> -#define vma_expand_down(vma, addr) (-EFAULT)
> -
> -#else
> -
> -#define vma_expand_up(vma,addr) (-EFAULT)
> -#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
> -
> -#endif
> -
> -/*
> - * expand_stack(): legacy interface for page faulting. Don't use unless
> - * you have to.
> - *
> - * This is called with the mm locked for reading, drops the lock, takes
> - * the lock for writing, tries to look up a vma again, expands it if
> - * necessary, and downgrades the lock to reading again.
> - *
> - * If no vma is found or it can't be expanded, it returns NULL and has
> - * dropped the lock.
> - */
> -struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
> -{
> -	struct vm_area_struct *vma, *prev;
> -
> -	mmap_read_unlock(mm);
> -	if (mmap_write_lock_killable(mm))
> -		return NULL;
> -
> -	vma = find_vma_prev(mm, addr, &prev);
> -	if (vma && vma->vm_start <= addr)
> -		goto success;
> -
> -	if (prev && !vma_expand_up(prev, addr)) {
> -		vma = prev;
> -		goto success;
> -	}
> -
> -	if (vma && !vma_expand_down(vma, addr))
> -		goto success;
> -
> -	mmap_write_unlock(mm);
> -	return NULL;
> -
> -success:
> -	mmap_write_downgrade(mm);
> -	return vma;
> -}
> -
> -/*
> - * Ok - we have the memory areas we should free on a maple tree so release them,
> - * and do the vma updates.
> - *
> - * Called with the mm semaphore held.
> - */
> -static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
> -{
> -	unsigned long nr_accounted = 0;
> -	struct vm_area_struct *vma;
> -
> -	/* Update high watermark before we lower total_vm */
> -	update_hiwater_vm(mm);
> -	mas_for_each(mas, vma, ULONG_MAX) {
> -		long nrpages = vma_pages(vma);
> -
> -		if (vma->vm_flags & VM_ACCOUNT)
> -			nr_accounted += nrpages;
> -		vm_stat_account(mm, vma->vm_flags, -nrpages);
> -		remove_vma(vma, false);
> -	}
> -	vm_unacct_memory(nr_accounted);
> -}
> -
> -/*
> - * Get rid of page table information in the indicated region.
> - *
> - * Called with the mm semaphore held.
> - */
> -static void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> -		struct vm_area_struct *vma, struct vm_area_struct *prev,
> -		struct vm_area_struct *next, unsigned long start,
> -		unsigned long end, unsigned long tree_end, bool mm_wr_locked)
> -{
> -	struct mmu_gather tlb;
> -	unsigned long mt_start = mas->index;
> -
> -	lru_add_drain();
> -	tlb_gather_mmu(&tlb, mm);
> -	update_hiwater_rss(mm);
> -	unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked);
> -	mas_set(mas, mt_start);
> -	free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
> -				 next ? next->vm_start : USER_PGTABLES_CEILING,
> -				 mm_wr_locked);
> -	tlb_finish_mmu(&tlb);
> -}
> -
> -/*
> - * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
> - * has already been checked or doesn't make sense to fail.
> - * VMA Iterator will point to the end VMA.
> - */
> -static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -		       unsigned long addr, int new_below)
> -{
> -	struct vma_prepare vp;
> -	struct vm_area_struct *new;
> -	int err;
> -
> -	WARN_ON(vma->vm_start >= addr);
> -	WARN_ON(vma->vm_end <= addr);
> -
> -	if (vma->vm_ops && vma->vm_ops->may_split) {
> -		err = vma->vm_ops->may_split(vma, addr);
> -		if (err)
> -			return err;
> -	}
> -
> -	new = vm_area_dup(vma);
> -	if (!new)
> -		return -ENOMEM;
> -
> -	if (new_below) {
> -		new->vm_end = addr;
> -	} else {
> -		new->vm_start = addr;
> -		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
> -	}
> -
> -	err = -ENOMEM;
> -	vma_iter_config(vmi, new->vm_start, new->vm_end);
> -	if (vma_iter_prealloc(vmi, new))
> -		goto out_free_vma;
> -
> -	err = vma_dup_policy(vma, new);
> -	if (err)
> -		goto out_free_vmi;
> -
> -	err = anon_vma_clone(new, vma);
> -	if (err)
> -		goto out_free_mpol;
> -
> -	if (new->vm_file)
> -		get_file(new->vm_file);
> -
> -	if (new->vm_ops && new->vm_ops->open)
> -		new->vm_ops->open(new);
> -
> -	vma_start_write(vma);
> -	vma_start_write(new);
> -
> -	init_vma_prep(&vp, vma);
> -	vp.insert = new;
> -	vma_prepare(&vp);
> -	vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
> -
> -	if (new_below) {
> -		vma->vm_start = addr;
> -		vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
> -	} else {
> -		vma->vm_end = addr;
> -	}
> -
> -	/* vma_complete stores the new vma */
> -	vma_complete(&vp, vmi, vma->vm_mm);
> -
> -	/* Success. */
> -	if (new_below)
> -		vma_next(vmi);
> -	return 0;
> -
> -out_free_mpol:
> -	mpol_put(vma_policy(new));
> -out_free_vmi:
> -	vma_iter_free(vmi);
> -out_free_vma:
> -	vm_area_free(new);
> -	return err;
> -}
> -
> -/*
> - * Split a vma into two pieces at address 'addr', a new vma is allocated
> - * either for the first part or the tail.
> - */
> -static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -		     unsigned long addr, int new_below)
> -{
> -	if (vma->vm_mm->map_count >= sysctl_max_map_count)
> -		return -ENOMEM;
> -
> -	return __split_vma(vmi, vma, addr, new_below);
> -}
> -
> -/*
> - * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
> - * context and anonymous VMA name within the range [start, end).
> - *
> - * As a result, we might be able to merge the newly modified VMA range with an
> - * adjacent VMA with identical properties.
> - *
> - * If no merge is possible and the range does not span the entirety of the VMA,
> - * we then need to split the VMA to accommodate the change.
> - *
> - * The function returns either the merged VMA, the original VMA if a split was
> - * required instead, or an error if the split failed.
> - */
> -struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> -				  struct vm_area_struct *prev,
> -				  struct vm_area_struct *vma,
> -				  unsigned long start, unsigned long end,
> -				  unsigned long vm_flags,
> -				  struct mempolicy *policy,
> -				  struct vm_userfaultfd_ctx uffd_ctx,
> -				  struct anon_vma_name *anon_name)
> -{
> -	pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> -	struct vm_area_struct *merged;
> -
> -	merged = vma_merge(vmi, prev, vma, start, end, vm_flags,
> -			   pgoff, policy, uffd_ctx, anon_name);
> -	if (merged)
> -		return merged;
> -
> -	if (vma->vm_start < start) {
> -		int err = split_vma(vmi, vma, start, 1);
> -
> -		if (err)
> -			return ERR_PTR(err);
> +				perf_event_mmap(vma);
> +			}
> +		}
>  	}
> +	anon_vma_unlock_write(vma->anon_vma);
> +	vma_iter_free(&vmi);
> +	validate_mm(mm);
> +	return error;
> +}
>  
> -	if (vma->vm_end > end) {
> -		int err = split_vma(vmi, vma, end, 0);
> +/* enforced gap between the expanding stack and other mappings. */
> +unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
>  
> -		if (err)
> -			return ERR_PTR(err);
> -	}
> +static int __init cmdline_parse_stack_guard_gap(char *p)
> +{
> +	unsigned long val;
> +	char *endptr;
>  
> -	return vma;
> +	val = simple_strtoul(p, &endptr, 10);
> +	if (!*endptr)
> +		stack_guard_gap = val << PAGE_SHIFT;
> +
> +	return 1;
>  }
> +__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
>  
> -/*
> - * Attempt to merge a newly mapped VMA with those adjacent to it. The caller
> - * must ensure that [start, end) does not overlap any existing VMA.
> - */
> -static struct vm_area_struct
> -*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
> -		   struct vm_area_struct *vma, unsigned long start,
> -		   unsigned long end, pgoff_t pgoff)
> +#ifdef CONFIG_STACK_GROWSUP
> +int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
>  {
> -	return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,
> -			 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> +	return expand_upwards(vma, address);
>  }
>  
> -/*
> - * Expand vma by delta bytes, potentially merging with an immediately adjacent
> - * VMA with identical properties.
> - */
> -struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> -					struct vm_area_struct *vma,
> -					unsigned long delta)
> +struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
>  {
> -	pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);
> +	struct vm_area_struct *vma, *prev;
>  
> -	/* vma is specified as prev, so case 1 or 2 will apply. */
> -	return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,
> -			 vma->vm_flags, pgoff, vma_policy(vma),
> -			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> +	addr &= PAGE_MASK;
> +	vma = find_vma_prev(mm, addr, &prev);
> +	if (vma && (vma->vm_start <= addr))
> +		return vma;
> +	if (!prev)
> +		return NULL;
> +	if (expand_stack_locked(prev, addr))
> +		return NULL;
> +	if (prev->vm_flags & VM_LOCKED)
> +		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
> +	return prev;
>  }
> -
> -/*
> - * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
> - * @vmi: The vma iterator
> - * @vma: The starting vm_area_struct
> - * @mm: The mm_struct
> - * @start: The aligned start address to munmap.
> - * @end: The aligned end address to munmap.
> - * @uf: The userfaultfd list_head
> - * @unlock: Set to true to drop the mmap_lock.  unlocking only happens on
> - * success.
> - *
> - * Return: 0 on success and drops the lock if so directed, error and leaves the
> - * lock held otherwise.
> - */
> -static int
> -do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
> -		    struct mm_struct *mm, unsigned long start,
> -		    unsigned long end, struct list_head *uf, bool unlock)
> +#else
> +int expand_stack_locked(struct vm_area_struct *vma, unsigned long address)
>  {
> -	struct vm_area_struct *prev, *next = NULL;
> -	struct maple_tree mt_detach;
> -	int count = 0;
> -	int error = -ENOMEM;
> -	unsigned long locked_vm = 0;
> -	MA_STATE(mas_detach, &mt_detach, 0, 0);
> -	mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
> -	mt_on_stack(mt_detach);
> -
> -	/*
> -	 * If we need to split any vma, do it now to save pain later.
> -	 *
> -	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
> -	 * unmapped vm_area_struct will remain in use: so lower split_vma
> -	 * places tmp vma above, and higher split_vma places tmp vma below.
> -	 */
> -
> -	/* Does it split the first one? */
> -	if (start > vma->vm_start) {
> -
> -		/*
> -		 * Make sure that map_count on return from munmap() will
> -		 * not exceed its limit; but let map_count go just above
> -		 * its limit temporarily, to help free resources as expected.
> -		 */
> -		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
> -			goto map_count_exceeded;
> -
> -		error = __split_vma(vmi, vma, start, 1);
> -		if (error)
> -			goto start_split_failed;
> -	}
> -
> -	/*
> -	 * Detach a range of VMAs from the mm. Using next as a temp variable as
> -	 * it is always overwritten.
> -	 */
> -	next = vma;
> -	do {
> -		/* Does it split the end? */
> -		if (next->vm_end > end) {
> -			error = __split_vma(vmi, next, end, 0);
> -			if (error)
> -				goto end_split_failed;
> -		}
> -		vma_start_write(next);
> -		mas_set(&mas_detach, count);
> -		error = mas_store_gfp(&mas_detach, next, GFP_KERNEL);
> -		if (error)
> -			goto munmap_gather_failed;
> -		vma_mark_detached(next, true);
> -		if (next->vm_flags & VM_LOCKED)
> -			locked_vm += vma_pages(next);
> +	return expand_downwards(vma, address);
> +}
>  
> -		count++;
> -		if (unlikely(uf)) {
> -			/*
> -			 * If userfaultfd_unmap_prep returns an error the vmas
> -			 * will remain split, but userland will get a
> -			 * highly unexpected error anyway. This is no
> -			 * different than the case where the first of the two
> -			 * __split_vma fails, but we don't undo the first
> -			 * split, despite we could. This is unlikely enough
> -			 * failure that it's not worth optimizing it for.
> -			 */
> -			error = userfaultfd_unmap_prep(next, start, end, uf);
> +struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr)
> +{
> +	struct vm_area_struct *vma;
> +	unsigned long start;
>  
> -			if (error)
> -				goto userfaultfd_error;
> -		}
> -#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> -		BUG_ON(next->vm_start < start);
> -		BUG_ON(next->vm_start > end);
> -#endif
> -	} for_each_vma_range(*vmi, next, end);
> -
> -#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> -	/* Make sure no VMAs are about to be lost. */
> -	{
> -		MA_STATE(test, &mt_detach, 0, 0);
> -		struct vm_area_struct *vma_mas, *vma_test;
> -		int test_count = 0;
> -
> -		vma_iter_set(vmi, start);
> -		rcu_read_lock();
> -		vma_test = mas_find(&test, count - 1);
> -		for_each_vma_range(*vmi, vma_mas, end) {
> -			BUG_ON(vma_mas != vma_test);
> -			test_count++;
> -			vma_test = mas_next(&test, count - 1);
> -		}
> -		rcu_read_unlock();
> -		BUG_ON(count != test_count);
> -	}
> +	addr &= PAGE_MASK;
> +	vma = find_vma(mm, addr);
> +	if (!vma)
> +		return NULL;
> +	if (vma->vm_start <= addr)
> +		return vma;
> +	start = vma->vm_start;
> +	if (expand_stack_locked(vma, addr))
> +		return NULL;
> +	if (vma->vm_flags & VM_LOCKED)
> +		populate_vma_page_range(vma, addr, start, NULL);
> +	return vma;
> +}
>  #endif
>  
> -	while (vma_iter_addr(vmi) > start)
> -		vma_iter_prev_range(vmi);
> -
> -	error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
> -	if (error)
> -		goto clear_tree_failed;
> -
> -	/* Point of no return */
> -	mm->locked_vm -= locked_vm;
> -	mm->map_count -= count;
> -	if (unlock)
> -		mmap_write_downgrade(mm);
> +#if defined(CONFIG_STACK_GROWSUP)
>  
> -	prev = vma_iter_prev_range(vmi);
> -	next = vma_next(vmi);
> -	if (next)
> -		vma_iter_prev_range(vmi);
> +#define vma_expand_up(vma,addr) expand_upwards(vma, addr)
> +#define vma_expand_down(vma, addr) (-EFAULT)
>  
> -	/*
> -	 * We can free page tables without write-locking mmap_lock because VMAs
> -	 * were isolated before we downgraded mmap_lock.
> -	 */
> -	mas_set(&mas_detach, 1);
> -	unmap_region(mm, &mas_detach, vma, prev, next, start, end, count,
> -		     !unlock);
> -	/* Statistics and freeing VMAs */
> -	mas_set(&mas_detach, 0);
> -	remove_mt(mm, &mas_detach);
> -	validate_mm(mm);
> -	if (unlock)
> -		mmap_read_unlock(mm);
> +#else
>  
> -	__mt_destroy(&mt_detach);
> -	return 0;
> +#define vma_expand_up(vma,addr) (-EFAULT)
> +#define vma_expand_down(vma, addr) expand_downwards(vma, addr)
>  
> -clear_tree_failed:
> -userfaultfd_error:
> -munmap_gather_failed:
> -end_split_failed:
> -	mas_set(&mas_detach, 0);
> -	mas_for_each(&mas_detach, next, end)
> -		vma_mark_detached(next, false);
> -
> -	__mt_destroy(&mt_detach);
> -start_split_failed:
> -map_count_exceeded:
> -	validate_mm(mm);
> -	return error;
> -}
> +#endif
>  
>  /*
> - * do_vmi_munmap() - munmap a given range.
> - * @vmi: The vma iterator
> - * @mm: The mm_struct
> - * @start: The start address to munmap
> - * @len: The length of the range to munmap
> - * @uf: The userfaultfd list_head
> - * @unlock: set to true if the user wants to drop the mmap_lock on success
> + * expand_stack(): legacy interface for page faulting. Don't use unless
> + * you have to.
>   *
> - * This function takes a @mas that is either pointing to the previous VMA or set
> - * to MA_START and sets it up to remove the mapping(s).  The @len will be
> - * aligned and any arch_unmap work will be preformed.
> + * This is called with the mm locked for reading, drops the lock, takes
> + * the lock for writing, tries to look up a vma again, expands it if
> + * necessary, and downgrades the lock to reading again.
>   *
> - * Return: 0 on success and drops the lock if so directed, error and leaves the
> - * lock held otherwise.
> + * If no vma is found or it can't be expanded, it returns NULL and has
> + * dropped the lock.
>   */
> -int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
> -		  unsigned long start, size_t len, struct list_head *uf,
> -		  bool unlock)
> +struct vm_area_struct *expand_stack(struct mm_struct *mm, unsigned long addr)
>  {
> -	unsigned long end;
> -	struct vm_area_struct *vma;
> +	struct vm_area_struct *vma, *prev;
>  
> -	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
> -		return -EINVAL;
> +	mmap_read_unlock(mm);
> +	if (mmap_write_lock_killable(mm))
> +		return NULL;
>  
> -	end = start + PAGE_ALIGN(len);
> -	if (end == start)
> -		return -EINVAL;
> +	vma = find_vma_prev(mm, addr, &prev);
> +	if (vma && vma->vm_start <= addr)
> +		goto success;
>  
> -	/*
> -	 * Check if memory is sealed before arch_unmap.
> -	 * Prevent unmapping a sealed VMA.
> -	 * can_modify_mm assumes we have acquired the lock on MM.
> -	 */
> -	if (unlikely(!can_modify_mm(mm, start, end)))
> -		return -EPERM;
> +	if (prev && !vma_expand_up(prev, addr)) {
> +		vma = prev;
> +		goto success;
> +	}
>  
> -	 /* arch_unmap() might do unmaps itself.  */
> -	arch_unmap(mm, start, end);
> +	if (vma && !vma_expand_down(vma, addr))
> +		goto success;
>  
> -	/* Find the first overlapping VMA */
> -	vma = vma_find(vmi, end);
> -	if (!vma) {
> -		if (unlock)
> -			mmap_write_unlock(mm);
> -		return 0;
> -	}
> +	mmap_write_unlock(mm);
> +	return NULL;
>  
> -	return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
> +success:
> +	mmap_write_downgrade(mm);
> +	return vma;
>  }
>  
>  /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls.
> @@ -3460,92 +1972,6 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
>  	return 0;
>  }
>  
> -/*
> - * Copy the vma structure to a new location in the same mm,
> - * prior to moving page table entries, to effect an mremap move.
> - */
> -struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> -	unsigned long addr, unsigned long len, pgoff_t pgoff,
> -	bool *need_rmap_locks)
> -{
> -	struct vm_area_struct *vma = *vmap;
> -	unsigned long vma_start = vma->vm_start;
> -	struct mm_struct *mm = vma->vm_mm;
> -	struct vm_area_struct *new_vma, *prev;
> -	bool faulted_in_anon_vma = true;
> -	VMA_ITERATOR(vmi, mm, addr);
> -
> -	/*
> -	 * If anonymous vma has not yet been faulted, update new pgoff
> -	 * to match new location, to increase its chance of merging.
> -	 */
> -	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
> -		pgoff = addr >> PAGE_SHIFT;
> -		faulted_in_anon_vma = false;
> -	}
> -
> -	new_vma = find_vma_prev(mm, addr, &prev);
> -	if (new_vma && new_vma->vm_start < addr + len)
> -		return NULL;	/* should never get here */
> -
> -	new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff);
> -	if (new_vma) {
> -		/*
> -		 * Source vma may have been merged into new_vma
> -		 */
> -		if (unlikely(vma_start >= new_vma->vm_start &&
> -			     vma_start < new_vma->vm_end)) {
> -			/*
> -			 * The only way we can get a vma_merge with
> -			 * self during an mremap is if the vma hasn't
> -			 * been faulted in yet and we were allowed to
> -			 * reset the dst vma->vm_pgoff to the
> -			 * destination address of the mremap to allow
> -			 * the merge to happen. mremap must change the
> -			 * vm_pgoff linearity between src and dst vmas
> -			 * (in turn preventing a vma_merge) to be
> -			 * safe. It is only safe to keep the vm_pgoff
> -			 * linear if there are no pages mapped yet.
> -			 */
> -			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
> -			*vmap = vma = new_vma;
> -		}
> -		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
> -	} else {
> -		new_vma = vm_area_dup(vma);
> -		if (!new_vma)
> -			goto out;
> -		vma_set_range(new_vma, addr, addr + len, pgoff);
> -		if (vma_dup_policy(vma, new_vma))
> -			goto out_free_vma;
> -		if (anon_vma_clone(new_vma, vma))
> -			goto out_free_mempol;
> -		if (new_vma->vm_file)
> -			get_file(new_vma->vm_file);
> -		if (new_vma->vm_ops && new_vma->vm_ops->open)
> -			new_vma->vm_ops->open(new_vma);
> -		if (vma_link(mm, new_vma))
> -			goto out_vma_link;
> -		*need_rmap_locks = false;
> -	}
> -	return new_vma;
> -
> -out_vma_link:
> -	if (new_vma->vm_ops && new_vma->vm_ops->close)
> -		new_vma->vm_ops->close(new_vma);
> -
> -	if (new_vma->vm_file)
> -		fput(new_vma->vm_file);
> -
> -	unlink_anon_vmas(new_vma);
> -out_free_mempol:
> -	mpol_put(vma_policy(new_vma));
> -out_free_vma:
> -	vm_area_free(new_vma);
> -out:
> -	return NULL;
> -}
> -
>  /*
>   * Return true if the calling process may expand its vm space by the passed
>   * number of pages
> @@ -3743,203 +2169,6 @@ int install_special_mapping(struct mm_struct *mm,
>  	return PTR_ERR_OR_ZERO(vma);
>  }
>  
> -static DEFINE_MUTEX(mm_all_locks_mutex);
> -
> -static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
> -{
> -	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> -		/*
> -		 * The LSB of head.next can't change from under us
> -		 * because we hold the mm_all_locks_mutex.
> -		 */
> -		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
> -		/*
> -		 * We can safely modify head.next after taking the
> -		 * anon_vma->root->rwsem. If some other vma in this mm shares
> -		 * the same anon_vma we won't take it again.
> -		 *
> -		 * No need of atomic instructions here, head.next
> -		 * can't change from under us thanks to the
> -		 * anon_vma->root->rwsem.
> -		 */
> -		if (__test_and_set_bit(0, (unsigned long *)
> -				       &anon_vma->root->rb_root.rb_root.rb_node))
> -			BUG();
> -	}
> -}
> -
> -static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
> -{
> -	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> -		/*
> -		 * AS_MM_ALL_LOCKS can't change from under us because
> -		 * we hold the mm_all_locks_mutex.
> -		 *
> -		 * Operations on ->flags have to be atomic because
> -		 * even if AS_MM_ALL_LOCKS is stable thanks to the
> -		 * mm_all_locks_mutex, there may be other cpus
> -		 * changing other bitflags in parallel to us.
> -		 */
> -		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
> -			BUG();
> -		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
> -	}
> -}
> -
> -/*
> - * This operation locks against the VM for all pte/vma/mm related
> - * operations that could ever happen on a certain mm. This includes
> - * vmtruncate, try_to_unmap, and all page faults.
> - *
> - * The caller must take the mmap_lock in write mode before calling
> - * mm_take_all_locks(). The caller isn't allowed to release the
> - * mmap_lock until mm_drop_all_locks() returns.
> - *
> - * mmap_lock in write mode is required in order to block all operations
> - * that could modify pagetables and free pages without need of
> - * altering the vma layout. It's also needed in write mode to avoid new
> - * anon_vmas to be associated with existing vmas.
> - *
> - * A single task can't take more than one mm_take_all_locks() in a row
> - * or it would deadlock.
> - *
> - * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
> - * mapping->flags avoid to take the same lock twice, if more than one
> - * vma in this mm is backed by the same anon_vma or address_space.
> - *
> - * We take locks in following order, accordingly to comment at beginning
> - * of mm/rmap.c:
> - *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
> - *     hugetlb mapping);
> - *   - all vmas marked locked
> - *   - all i_mmap_rwsem locks;
> - *   - all anon_vma->rwseml
> - *
> - * We can take all locks within these types randomly because the VM code
> - * doesn't nest them and we protected from parallel mm_take_all_locks() by
> - * mm_all_locks_mutex.
> - *
> - * mm_take_all_locks() and mm_drop_all_locks are expensive operations
> - * that may have to take thousand of locks.
> - *
> - * mm_take_all_locks() can fail if it's interrupted by signals.
> - */
> -int mm_take_all_locks(struct mm_struct *mm)
> -{
> -	struct vm_area_struct *vma;
> -	struct anon_vma_chain *avc;
> -	VMA_ITERATOR(vmi, mm, 0);
> -
> -	mmap_assert_write_locked(mm);
> -
> -	mutex_lock(&mm_all_locks_mutex);
> -
> -	/*
> -	 * vma_start_write() does not have a complement in mm_drop_all_locks()
> -	 * because vma_start_write() is always asymmetrical; it marks a VMA as
> -	 * being written to until mmap_write_unlock() or mmap_write_downgrade()
> -	 * is reached.
> -	 */
> -	for_each_vma(vmi, vma) {
> -		if (signal_pending(current))
> -			goto out_unlock;
> -		vma_start_write(vma);
> -	}
> -
> -	vma_iter_init(&vmi, mm, 0);
> -	for_each_vma(vmi, vma) {
> -		if (signal_pending(current))
> -			goto out_unlock;
> -		if (vma->vm_file && vma->vm_file->f_mapping &&
> -				is_vm_hugetlb_page(vma))
> -			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> -	}
> -
> -	vma_iter_init(&vmi, mm, 0);
> -	for_each_vma(vmi, vma) {
> -		if (signal_pending(current))
> -			goto out_unlock;
> -		if (vma->vm_file && vma->vm_file->f_mapping &&
> -				!is_vm_hugetlb_page(vma))
> -			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> -	}
> -
> -	vma_iter_init(&vmi, mm, 0);
> -	for_each_vma(vmi, vma) {
> -		if (signal_pending(current))
> -			goto out_unlock;
> -		if (vma->anon_vma)
> -			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> -				vm_lock_anon_vma(mm, avc->anon_vma);
> -	}
> -
> -	return 0;
> -
> -out_unlock:
> -	mm_drop_all_locks(mm);
> -	return -EINTR;
> -}
> -
> -static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
> -{
> -	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> -		/*
> -		 * The LSB of head.next can't change to 0 from under
> -		 * us because we hold the mm_all_locks_mutex.
> -		 *
> -		 * We must however clear the bitflag before unlocking
> -		 * the vma so the users using the anon_vma->rb_root will
> -		 * never see our bitflag.
> -		 *
> -		 * No need of atomic instructions here, head.next
> -		 * can't change from under us until we release the
> -		 * anon_vma->root->rwsem.
> -		 */
> -		if (!__test_and_clear_bit(0, (unsigned long *)
> -					  &anon_vma->root->rb_root.rb_root.rb_node))
> -			BUG();
> -		anon_vma_unlock_write(anon_vma);
> -	}
> -}
> -
> -static void vm_unlock_mapping(struct address_space *mapping)
> -{
> -	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> -		/*
> -		 * AS_MM_ALL_LOCKS can't change to 0 from under us
> -		 * because we hold the mm_all_locks_mutex.
> -		 */
> -		i_mmap_unlock_write(mapping);
> -		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
> -					&mapping->flags))
> -			BUG();
> -	}
> -}
> -
> -/*
> - * The mmap_lock cannot be released by the caller until
> - * mm_drop_all_locks() returns.
> - */
> -void mm_drop_all_locks(struct mm_struct *mm)
> -{
> -	struct vm_area_struct *vma;
> -	struct anon_vma_chain *avc;
> -	VMA_ITERATOR(vmi, mm, 0);
> -
> -	mmap_assert_write_locked(mm);
> -	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
> -
> -	for_each_vma(vmi, vma) {
> -		if (vma->anon_vma)
> -			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> -				vm_unlock_anon_vma(avc->anon_vma);
> -		if (vma->vm_file && vma->vm_file->f_mapping)
> -			vm_unlock_mapping(vma->vm_file->f_mapping);
> -	}
> -
> -	mutex_unlock(&mm_all_locks_mutex);
> -}
> -
>  /*
>   * vma_expand_bottom() - Expands the bottom of a VMA downwards. An error will
>   *                       arise if there is another VMA in the expanded range, or
> diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
> index 8982e6139d07..fc18fe274505 100644
> --- a/mm/mmu_notifier.c
> +++ b/mm/mmu_notifier.c
> @@ -19,6 +19,8 @@
>  #include <linux/sched/mm.h>
>  #include <linux/slab.h>
>  
> +#include "vma.h"
> +
>  /* global SRCU for all MMs */
>  DEFINE_STATIC_SRCU(srcu);
>  
> diff --git a/mm/mprotect.c b/mm/mprotect.c
> index 222ab434da54..77951e2d0863 100644
> --- a/mm/mprotect.c
> +++ b/mm/mprotect.c
> @@ -39,6 +39,7 @@
>  #include <asm/tlb.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
>  			     pte_t pte)
> diff --git a/mm/mremap.c b/mm/mremap.c
> index e7ae140fc640..09ef3eb31fbf 100644
> --- a/mm/mremap.c
> +++ b/mm/mremap.c
> @@ -31,6 +31,7 @@
>  #include <asm/pgalloc.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr)
>  {
> diff --git a/mm/mseal.c b/mm/mseal.c
> index bf783bba8ed0..7bcceda42a1a 100644
> --- a/mm/mseal.c
> +++ b/mm/mseal.c
> @@ -14,7 +14,9 @@
>  #include <linux/mmu_context.h>
>  #include <linux/syscalls.h>
>  #include <linux/sched.h>
> +
>  #include "internal.h"
> +#include "vma.h"
>  
>  static inline bool vma_is_sealed(struct vm_area_struct *vma)
>  {
> diff --git a/mm/rmap.c b/mm/rmap.c
> index 8616308610b9..4dec7ab3638c 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -83,6 +83,7 @@
>  #include <trace/events/migrate.h>
>  
>  #include "internal.h"
> +#include "vma.h"
>  
>  static struct kmem_cache *anon_vma_cachep;
>  static struct kmem_cache *anon_vma_chain_cachep;
> diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
> index 950fe6b2f0f7..30be083788be 100644
> --- a/mm/userfaultfd.c
> +++ b/mm/userfaultfd.c
> @@ -17,7 +17,9 @@
>  #include <linux/shmem_fs.h>
>  #include <asm/tlbflush.h>
>  #include <asm/tlb.h>
> +
>  #include "internal.h"
> +#include "vma.h"
>  
>  static __always_inline
>  bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end)
> diff --git a/mm/vma.c b/mm/vma.c
> new file mode 100644
> index 000000000000..bf0546fe6eab
> --- /dev/null
> +++ b/mm/vma.c
> @@ -0,0 +1,1766 @@
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +
> +/*
> + * VMA-specific functions.
> + */
> +
> +#include "vma_internal.h"
> +#include "vma.h"
> +
> +/*
> + * If the vma has a ->close operation then the driver probably needs to release
> + * per-vma resources, so we don't attempt to merge those if the caller indicates
> + * the current vma may be removed as part of the merge.
> + */
> +static inline bool is_mergeable_vma(struct vm_area_struct *vma,
> +		struct file *file, unsigned long vm_flags,
> +		struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> +		struct anon_vma_name *anon_name, bool may_remove_vma)
> +{
> +	/*
> +	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
> +	 * match the flags but dirty bit -- the caller should mark
> +	 * merged VMA as dirty. If dirty bit won't be excluded from
> +	 * comparison, we increase pressure on the memory system forcing
> +	 * the kernel to generate new VMAs when old one could be
> +	 * extended instead.
> +	 */
> +	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
> +		return false;
> +	if (vma->vm_file != file)
> +		return false;
> +	if (may_remove_vma && vma->vm_ops && vma->vm_ops->close)
> +		return false;
> +	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
> +		return false;
> +	if (!anon_vma_name_eq(anon_vma_name(vma), anon_name))
> +		return false;
> +	return true;
> +}
> +
> +static inline bool is_mergeable_anon_vma(struct anon_vma *anon_vma1,
> +		 struct anon_vma *anon_vma2, struct vm_area_struct *vma)
> +{
> +	/*
> +	 * The list_is_singular() test is to avoid merging VMA cloned from
> +	 * parents. This can improve scalability caused by anon_vma lock.
> +	 */
> +	if ((!anon_vma1 || !anon_vma2) && (!vma ||
> +		list_is_singular(&vma->anon_vma_chain)))
> +		return true;
> +	return anon_vma1 == anon_vma2;
> +}
> +
> +/*
> + * init_multi_vma_prep() - Initializer for struct vma_prepare
> + * @vp: The vma_prepare struct
> + * @vma: The vma that will be altered once locked
> + * @next: The next vma if it is to be adjusted
> + * @remove: The first vma to be removed
> + * @remove2: The second vma to be removed
> + */
> +static void init_multi_vma_prep(struct vma_prepare *vp,
> +				struct vm_area_struct *vma,
> +				struct vm_area_struct *next,
> +				struct vm_area_struct *remove,
> +				struct vm_area_struct *remove2)
> +{
> +	memset(vp, 0, sizeof(struct vma_prepare));
> +	vp->vma = vma;
> +	vp->anon_vma = vma->anon_vma;
> +	vp->remove = remove;
> +	vp->remove2 = remove2;
> +	vp->adj_next = next;
> +	if (!vp->anon_vma && next)
> +		vp->anon_vma = next->anon_vma;
> +
> +	vp->file = vma->vm_file;
> +	if (vp->file)
> +		vp->mapping = vma->vm_file->f_mapping;
> +
> +}
> +
> +/*
> + * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> + * in front of (at a lower virtual address and file offset than) the vma.
> + *
> + * We cannot merge two vmas if they have differently assigned (non-NULL)
> + * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> + *
> + * We don't check here for the merged mmap wrapping around the end of pagecache
> + * indices (16TB on ia32) because do_mmap() does not permit mmap's which
> + * wrap, nor mmaps which cover the final page at index -1UL.
> + *
> + * We assume the vma may be removed as part of the merge.
> + */
> +bool
> +can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
> +		struct anon_vma *anon_vma, struct file *file,
> +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> +		struct anon_vma_name *anon_name)
> +{
> +	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, true) &&
> +	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> +		if (vma->vm_pgoff == vm_pgoff)
> +			return true;
> +	}
> +	return false;
> +}
> +
> +/*
> + * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
> + * beyond (at a higher virtual address and file offset than) the vma.
> + *
> + * We cannot merge two vmas if they have differently assigned (non-NULL)
> + * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
> + *
> + * We assume that vma is not removed as part of the merge.
> + */
> +bool
> +can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
> +		struct anon_vma *anon_vma, struct file *file,
> +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> +		struct anon_vma_name *anon_name)
> +{
> +	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx, anon_name, false) &&
> +	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
> +		pgoff_t vm_pglen;
> +
> +		vm_pglen = vma_pages(vma);
> +		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
> +			return true;
> +	}
> +	return false;
> +}
> +
> +/*
> + * Close a vm structure and free it.
> + */
> +void remove_vma(struct vm_area_struct *vma, bool unreachable)
> +{
> +	might_sleep();
> +	if (vma->vm_ops && vma->vm_ops->close)
> +		vma->vm_ops->close(vma);
> +	if (vma->vm_file)
> +		fput(vma->vm_file);
> +	mpol_put(vma_policy(vma));
> +	if (unreachable)
> +		__vm_area_free(vma);
> +	else
> +		vm_area_free(vma);
> +}
> +
> +/*
> + * Get rid of page table information in the indicated region.
> + *
> + * Called with the mm semaphore held.
> + */
> +void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> +		struct vm_area_struct *vma, struct vm_area_struct *prev,
> +		struct vm_area_struct *next, unsigned long start,
> +		unsigned long end, unsigned long tree_end, bool mm_wr_locked)
> +{
> +	struct mmu_gather tlb;
> +	unsigned long mt_start = mas->index;
> +
> +	lru_add_drain();
> +	tlb_gather_mmu(&tlb, mm);
> +	update_hiwater_rss(mm);
> +	unmap_vmas(&tlb, mas, vma, start, end, tree_end, mm_wr_locked);
> +	mas_set(mas, mt_start);
> +	free_pgtables(&tlb, mas, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
> +				 next ? next->vm_start : USER_PGTABLES_CEILING,
> +				 mm_wr_locked);
> +	tlb_finish_mmu(&tlb);
> +}
> +
> +/*
> + * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
> + * has already been checked or doesn't make sense to fail.
> + * VMA Iterator will point to the end VMA.
> + */
> +static int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +		       unsigned long addr, int new_below)
> +{
> +	struct vma_prepare vp;
> +	struct vm_area_struct *new;
> +	int err;
> +
> +	WARN_ON(vma->vm_start >= addr);
> +	WARN_ON(vma->vm_end <= addr);
> +
> +	if (vma->vm_ops && vma->vm_ops->may_split) {
> +		err = vma->vm_ops->may_split(vma, addr);
> +		if (err)
> +			return err;
> +	}
> +
> +	new = vm_area_dup(vma);
> +	if (!new)
> +		return -ENOMEM;
> +
> +	if (new_below) {
> +		new->vm_end = addr;
> +	} else {
> +		new->vm_start = addr;
> +		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
> +	}
> +
> +	err = -ENOMEM;
> +	vma_iter_config(vmi, new->vm_start, new->vm_end);
> +	if (vma_iter_prealloc(vmi, new))
> +		goto out_free_vma;
> +
> +	err = vma_dup_policy(vma, new);
> +	if (err)
> +		goto out_free_vmi;
> +
> +	err = anon_vma_clone(new, vma);
> +	if (err)
> +		goto out_free_mpol;
> +
> +	if (new->vm_file)
> +		get_file(new->vm_file);
> +
> +	if (new->vm_ops && new->vm_ops->open)
> +		new->vm_ops->open(new);
> +
> +	vma_start_write(vma);
> +	vma_start_write(new);
> +
> +	init_vma_prep(&vp, vma);
> +	vp.insert = new;
> +	vma_prepare(&vp);
> +	vma_adjust_trans_huge(vma, vma->vm_start, addr, 0);
> +
> +	if (new_below) {
> +		vma->vm_start = addr;
> +		vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT;
> +	} else {
> +		vma->vm_end = addr;
> +	}
> +
> +	/* vma_complete stores the new vma */
> +	vma_complete(&vp, vmi, vma->vm_mm);
> +
> +	/* Success. */
> +	if (new_below)
> +		vma_next(vmi);
> +	return 0;
> +
> +out_free_mpol:
> +	mpol_put(vma_policy(new));
> +out_free_vmi:
> +	vma_iter_free(vmi);
> +out_free_vma:
> +	vm_area_free(new);
> +	return err;
> +}
> +
> +/*
> + * Split a vma into two pieces at address 'addr', a new vma is allocated
> + * either for the first part or the tail.
> + */
> +static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +		     unsigned long addr, int new_below)
> +{
> +	if (vma->vm_mm->map_count >= sysctl_max_map_count)
> +		return -ENOMEM;
> +
> +	return __split_vma(vmi, vma, addr, new_below);
> +}
> +
> +/*
> + * Ok - we have the memory areas we should free on a maple tree so release them,
> + * and do the vma updates.
> + *
> + * Called with the mm semaphore held.
> + */
> +static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
> +{
> +	unsigned long nr_accounted = 0;
> +	struct vm_area_struct *vma;
> +
> +	/* Update high watermark before we lower total_vm */
> +	update_hiwater_vm(mm);
> +	mas_for_each(mas, vma, ULONG_MAX) {
> +		long nrpages = vma_pages(vma);
> +
> +		if (vma->vm_flags & VM_ACCOUNT)
> +			nr_accounted += nrpages;
> +		vm_stat_account(mm, vma->vm_flags, -nrpages);
> +		remove_vma(vma, false);
> +	}
> +	vm_unacct_memory(nr_accounted);
> +}
> +
> +/*
> + * init_vma_prep() - Initializer wrapper for vma_prepare struct
> + * @vp: The vma_prepare struct
> + * @vma: The vma that will be altered once locked
> + */
> +void init_vma_prep(struct vma_prepare *vp,
> +		   struct vm_area_struct *vma)
> +{
> +	init_multi_vma_prep(vp, vma, NULL, NULL, NULL);
> +}
> +
> +/*
> + * Requires inode->i_mapping->i_mmap_rwsem
> + */
> +static void __remove_shared_vm_struct(struct vm_area_struct *vma,
> +				      struct address_space *mapping)
> +{
> +	if (vma_is_shared_maywrite(vma))
> +		mapping_unmap_writable(mapping);
> +
> +	flush_dcache_mmap_lock(mapping);
> +	vma_interval_tree_remove(vma, &mapping->i_mmap);
> +	flush_dcache_mmap_unlock(mapping);
> +}
> +
> +/*
> + * vma has some anon_vma assigned, and is already inserted on that
> + * anon_vma's interval trees.
> + *
> + * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
> + * vma must be removed from the anon_vma's interval trees using
> + * anon_vma_interval_tree_pre_update_vma().
> + *
> + * After the update, the vma will be reinserted using
> + * anon_vma_interval_tree_post_update_vma().
> + *
> + * The entire update must be protected by exclusive mmap_lock and by
> + * the root anon_vma's mutex.
> + */
> +void
> +anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
> +{
> +	struct anon_vma_chain *avc;
> +
> +	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> +		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
> +}
> +
> +void
> +anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
> +{
> +	struct anon_vma_chain *avc;
> +
> +	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> +		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
> +}
> +
> +static void __vma_link_file(struct vm_area_struct *vma,
> +			    struct address_space *mapping)
> +{
> +	if (vma_is_shared_maywrite(vma))
> +		mapping_allow_writable(mapping);
> +
> +	flush_dcache_mmap_lock(mapping);
> +	vma_interval_tree_insert(vma, &mapping->i_mmap);
> +	flush_dcache_mmap_unlock(mapping);
> +}
> +
> +/*
> + * vma_prepare() - Helper function for handling locking VMAs prior to altering
> + * @vp: The initialized vma_prepare struct
> + */
> +void vma_prepare(struct vma_prepare *vp)
> +{
> +	if (vp->file) {
> +		uprobe_munmap(vp->vma, vp->vma->vm_start, vp->vma->vm_end);
> +
> +		if (vp->adj_next)
> +			uprobe_munmap(vp->adj_next, vp->adj_next->vm_start,
> +				      vp->adj_next->vm_end);
> +
> +		i_mmap_lock_write(vp->mapping);
> +		if (vp->insert && vp->insert->vm_file) {
> +			/*
> +			 * Put into interval tree now, so instantiated pages
> +			 * are visible to arm/parisc __flush_dcache_page
> +			 * throughout; but we cannot insert into address
> +			 * space until vma start or end is updated.
> +			 */
> +			__vma_link_file(vp->insert,
> +					vp->insert->vm_file->f_mapping);
> +		}
> +	}
> +
> +	if (vp->anon_vma) {
> +		anon_vma_lock_write(vp->anon_vma);
> +		anon_vma_interval_tree_pre_update_vma(vp->vma);
> +		if (vp->adj_next)
> +			anon_vma_interval_tree_pre_update_vma(vp->adj_next);
> +	}
> +
> +	if (vp->file) {
> +		flush_dcache_mmap_lock(vp->mapping);
> +		vma_interval_tree_remove(vp->vma, &vp->mapping->i_mmap);
> +		if (vp->adj_next)
> +			vma_interval_tree_remove(vp->adj_next,
> +						 &vp->mapping->i_mmap);
> +	}
> +
> +}
> +
> +/*
> + * dup_anon_vma() - Helper function to duplicate anon_vma
> + * @dst: The destination VMA
> + * @src: The source VMA
> + * @dup: Pointer to the destination VMA when successful.
> + *
> + * Returns: 0 on success.
> + */
> +static int dup_anon_vma(struct vm_area_struct *dst,
> +			struct vm_area_struct *src, struct vm_area_struct **dup)
> +{
> +	/*
> +	 * Easily overlooked: when mprotect shifts the boundary, make sure the
> +	 * expanding vma has anon_vma set if the shrinking vma had, to cover any
> +	 * anon pages imported.
> +	 */
> +	if (src->anon_vma && !dst->anon_vma) {
> +		int ret;
> +
> +		vma_assert_write_locked(dst);
> +		dst->anon_vma = src->anon_vma;
> +		ret = anon_vma_clone(dst, src);
> +		if (ret)
> +			return ret;
> +
> +		*dup = dst;
> +	}
> +
> +	return 0;
> +}
> +
> +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> +void validate_mm(struct mm_struct *mm)
> +{
> +	int bug = 0;
> +	int i = 0;
> +	struct vm_area_struct *vma;
> +	VMA_ITERATOR(vmi, mm, 0);
> +
> +	mt_validate(&mm->mm_mt);
> +	for_each_vma(vmi, vma) {
> +#ifdef CONFIG_DEBUG_VM_RB
> +		struct anon_vma *anon_vma = vma->anon_vma;
> +		struct anon_vma_chain *avc;
> +#endif
> +		unsigned long vmi_start, vmi_end;
> +		bool warn = 0;
> +
> +		vmi_start = vma_iter_addr(&vmi);
> +		vmi_end = vma_iter_end(&vmi);
> +		if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
> +			warn = 1;
> +
> +		if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))
> +			warn = 1;
> +
> +		if (warn) {
> +			pr_emerg("issue in %s\n", current->comm);
> +			dump_stack();
> +			dump_vma(vma);
> +			pr_emerg("tree range: %px start %lx end %lx\n", vma,
> +				 vmi_start, vmi_end - 1);
> +			vma_iter_dump_tree(&vmi);
> +		}
> +
> +#ifdef CONFIG_DEBUG_VM_RB
> +		if (anon_vma) {
> +			anon_vma_lock_read(anon_vma);
> +			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> +				anon_vma_interval_tree_verify(avc);
> +			anon_vma_unlock_read(anon_vma);
> +		}
> +#endif
> +		i++;
> +	}
> +	if (i != mm->map_count) {
> +		pr_emerg("map_count %d vma iterator %d\n", mm->map_count, i);
> +		bug = 1;
> +	}
> +	VM_BUG_ON_MM(bug, mm);
> +}
> +#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */
> +
> +/*
> + * vma_expand - Expand an existing VMA
> + *
> + * @vmi: The vma iterator
> + * @vma: The vma to expand
> + * @start: The start of the vma
> + * @end: The exclusive end of the vma
> + * @pgoff: The page offset of vma
> + * @next: The current of next vma.
> + *
> + * Expand @vma to @start and @end.  Can expand off the start and end.  Will
> + * expand over @next if it's different from @vma and @end == @next->vm_end.
> + * Checking if the @vma can expand and merge with @next needs to be handled by
> + * the caller.
> + *
> + * Returns: 0 on success
> + */
> +int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +	       unsigned long start, unsigned long end, pgoff_t pgoff,
> +	       struct vm_area_struct *next)
> +{
> +	struct vm_area_struct *anon_dup = NULL;
> +	bool remove_next = false;
> +	struct vma_prepare vp;
> +
> +	vma_start_write(vma);
> +	if (next && (vma != next) && (end == next->vm_end)) {
> +		int ret;
> +
> +		remove_next = true;
> +		vma_start_write(next);
> +		ret = dup_anon_vma(vma, next, &anon_dup);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	init_multi_vma_prep(&vp, vma, NULL, remove_next ? next : NULL, NULL);
> +	/* Not merging but overwriting any part of next is not handled. */
> +	VM_WARN_ON(next && !vp.remove &&
> +		  next != vma && end > next->vm_start);
> +	/* Only handles expanding */
> +	VM_WARN_ON(vma->vm_start < start || vma->vm_end > end);
> +
> +	/* Note: vma iterator must be pointing to 'start' */
> +	vma_iter_config(vmi, start, end);
> +	if (vma_iter_prealloc(vmi, vma))
> +		goto nomem;
> +
> +	vma_prepare(&vp);
> +	vma_adjust_trans_huge(vma, start, end, 0);
> +	vma_set_range(vma, start, end, pgoff);
> +	vma_iter_store(vmi, vma);
> +
> +	vma_complete(&vp, vmi, vma->vm_mm);
> +	return 0;
> +
> +nomem:
> +	if (anon_dup)
> +		unlink_anon_vmas(anon_dup);
> +	return -ENOMEM;
> +}
> +
> +/*
> + * vma_shrink() - Reduce an existing VMAs memory area
> + * @vmi: The vma iterator
> + * @vma: The VMA to modify
> + * @start: The new start
> + * @end: The new end
> + *
> + * Returns: 0 on success, -ENOMEM otherwise
> + */
> +int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +	       unsigned long start, unsigned long end, pgoff_t pgoff)
> +{
> +	struct vma_prepare vp;
> +
> +	WARN_ON((vma->vm_start != start) && (vma->vm_end != end));
> +
> +	if (vma->vm_start < start)
> +		vma_iter_config(vmi, vma->vm_start, start);
> +	else
> +		vma_iter_config(vmi, end, vma->vm_end);
> +
> +	if (vma_iter_prealloc(vmi, NULL))
> +		return -ENOMEM;
> +
> +	vma_start_write(vma);
> +
> +	init_vma_prep(&vp, vma);
> +	vma_prepare(&vp);
> +	vma_adjust_trans_huge(vma, start, end, 0);
> +
> +	vma_iter_clear(vmi);
> +	vma_set_range(vma, start, end, pgoff);
> +	vma_complete(&vp, vmi, vma->vm_mm);
> +	return 0;
> +}
> +
> +/*
> + * vma_complete- Helper function for handling the unlocking after altering VMAs,
> + * or for inserting a VMA.
> + *
> + * @vp: The vma_prepare struct
> + * @vmi: The vma iterator
> + * @mm: The mm_struct
> + */
> +void vma_complete(struct vma_prepare *vp,
> +		  struct vma_iterator *vmi, struct mm_struct *mm)
> +{
> +	if (vp->file) {
> +		if (vp->adj_next)
> +			vma_interval_tree_insert(vp->adj_next,
> +						 &vp->mapping->i_mmap);
> +		vma_interval_tree_insert(vp->vma, &vp->mapping->i_mmap);
> +		flush_dcache_mmap_unlock(vp->mapping);
> +	}
> +
> +	if (vp->remove && vp->file) {
> +		__remove_shared_vm_struct(vp->remove, vp->mapping);
> +		if (vp->remove2)
> +			__remove_shared_vm_struct(vp->remove2, vp->mapping);
> +	} else if (vp->insert) {
> +		/*
> +		 * split_vma has split insert from vma, and needs
> +		 * us to insert it before dropping the locks
> +		 * (it may either follow vma or precede it).
> +		 */
> +		vma_iter_store(vmi, vp->insert);
> +		mm->map_count++;
> +	}
> +
> +	if (vp->anon_vma) {
> +		anon_vma_interval_tree_post_update_vma(vp->vma);
> +		if (vp->adj_next)
> +			anon_vma_interval_tree_post_update_vma(vp->adj_next);
> +		anon_vma_unlock_write(vp->anon_vma);
> +	}
> +
> +	if (vp->file) {
> +		i_mmap_unlock_write(vp->mapping);
> +		uprobe_mmap(vp->vma);
> +
> +		if (vp->adj_next)
> +			uprobe_mmap(vp->adj_next);
> +	}
> +
> +	if (vp->remove) {
> +again:
> +		vma_mark_detached(vp->remove, true);
> +		if (vp->file) {
> +			uprobe_munmap(vp->remove, vp->remove->vm_start,
> +				      vp->remove->vm_end);
> +			fput(vp->file);
> +		}
> +		if (vp->remove->anon_vma)
> +			anon_vma_merge(vp->vma, vp->remove);
> +		mm->map_count--;
> +		mpol_put(vma_policy(vp->remove));
> +		if (!vp->remove2)
> +			WARN_ON_ONCE(vp->vma->vm_end < vp->remove->vm_end);
> +		vm_area_free(vp->remove);
> +
> +		/*
> +		 * In mprotect's case 6 (see comments on vma_merge),
> +		 * we are removing both mid and next vmas
> +		 */
> +		if (vp->remove2) {
> +			vp->remove = vp->remove2;
> +			vp->remove2 = NULL;
> +			goto again;
> +		}
> +	}
> +	if (vp->insert && vp->file)
> +		uprobe_mmap(vp->insert);
> +	validate_mm(mm);
> +}
> +
> +/*
> + * do_vmi_align_munmap() - munmap the aligned region from @start to @end.
> + * @vmi: The vma iterator
> + * @vma: The starting vm_area_struct
> + * @mm: The mm_struct
> + * @start: The aligned start address to munmap.
> + * @end: The aligned end address to munmap.
> + * @uf: The userfaultfd list_head
> + * @unlock: Set to true to drop the mmap_lock.  unlocking only happens on
> + * success.
> + *
> + * Return: 0 on success and drops the lock if so directed, error and leaves the
> + * lock held otherwise.
> + */
> +int
> +do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +		    struct mm_struct *mm, unsigned long start,
> +		    unsigned long end, struct list_head *uf, bool unlock)
> +{
> +	struct vm_area_struct *prev, *next = NULL;
> +	struct maple_tree mt_detach;
> +	int count = 0;
> +	int error = -ENOMEM;
> +	unsigned long locked_vm = 0;
> +	MA_STATE(mas_detach, &mt_detach, 0, 0);
> +	mt_init_flags(&mt_detach, vmi->mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
> +	mt_on_stack(mt_detach);
> +
> +	/*
> +	 * If we need to split any vma, do it now to save pain later.
> +	 *
> +	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
> +	 * unmapped vm_area_struct will remain in use: so lower split_vma
> +	 * places tmp vma above, and higher split_vma places tmp vma below.
> +	 */
> +
> +	/* Does it split the first one? */
> +	if (start > vma->vm_start) {
> +
> +		/*
> +		 * Make sure that map_count on return from munmap() will
> +		 * not exceed its limit; but let map_count go just above
> +		 * its limit temporarily, to help free resources as expected.
> +		 */
> +		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
> +			goto map_count_exceeded;
> +
> +		error = __split_vma(vmi, vma, start, 1);
> +		if (error)
> +			goto start_split_failed;
> +	}
> +
> +	/*
> +	 * Detach a range of VMAs from the mm. Using next as a temp variable as
> +	 * it is always overwritten.
> +	 */
> +	next = vma;
> +	do {
> +		/* Does it split the end? */
> +		if (next->vm_end > end) {
> +			error = __split_vma(vmi, next, end, 0);
> +			if (error)
> +				goto end_split_failed;
> +		}
> +		vma_start_write(next);
> +		mas_set(&mas_detach, count);
> +		error = mas_store_gfp(&mas_detach, next, GFP_KERNEL);
> +		if (error)
> +			goto munmap_gather_failed;
> +		vma_mark_detached(next, true);
> +		if (next->vm_flags & VM_LOCKED)
> +			locked_vm += vma_pages(next);
> +
> +		count++;
> +		if (unlikely(uf)) {
> +			/*
> +			 * If userfaultfd_unmap_prep returns an error the vmas
> +			 * will remain split, but userland will get a
> +			 * highly unexpected error anyway. This is no
> +			 * different than the case where the first of the two
> +			 * __split_vma fails, but we don't undo the first
> +			 * split, despite we could. This is unlikely enough
> +			 * failure that it's not worth optimizing it for.
> +			 */
> +			error = userfaultfd_unmap_prep(next, start, end, uf);
> +
> +			if (error)
> +				goto userfaultfd_error;
> +		}
> +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> +		BUG_ON(next->vm_start < start);
> +		BUG_ON(next->vm_start > end);
> +#endif
> +	} for_each_vma_range(*vmi, next, end);
> +
> +#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> +	/* Make sure no VMAs are about to be lost. */
> +	{
> +		MA_STATE(test, &mt_detach, 0, 0);
> +		struct vm_area_struct *vma_mas, *vma_test;
> +		int test_count = 0;
> +
> +		vma_iter_set(vmi, start);
> +		rcu_read_lock();
> +		vma_test = mas_find(&test, count - 1);
> +		for_each_vma_range(*vmi, vma_mas, end) {
> +			BUG_ON(vma_mas != vma_test);
> +			test_count++;
> +			vma_test = mas_next(&test, count - 1);
> +		}
> +		rcu_read_unlock();
> +		BUG_ON(count != test_count);
> +	}
> +#endif
> +
> +	while (vma_iter_addr(vmi) > start)
> +		vma_iter_prev_range(vmi);
> +
> +	error = vma_iter_clear_gfp(vmi, start, end, GFP_KERNEL);
> +	if (error)
> +		goto clear_tree_failed;
> +
> +	/* Point of no return */
> +	mm->locked_vm -= locked_vm;
> +	mm->map_count -= count;
> +	if (unlock)
> +		mmap_write_downgrade(mm);
> +
> +	prev = vma_iter_prev_range(vmi);
> +	next = vma_next(vmi);
> +	if (next)
> +		vma_iter_prev_range(vmi);
> +
> +	/*
> +	 * We can free page tables without write-locking mmap_lock because VMAs
> +	 * were isolated before we downgraded mmap_lock.
> +	 */
> +	mas_set(&mas_detach, 1);
> +	unmap_region(mm, &mas_detach, vma, prev, next, start, end, count,
> +		     !unlock);
> +	/* Statistics and freeing VMAs */
> +	mas_set(&mas_detach, 0);
> +	remove_mt(mm, &mas_detach);
> +	validate_mm(mm);
> +	if (unlock)
> +		mmap_read_unlock(mm);
> +
> +	__mt_destroy(&mt_detach);
> +	return 0;
> +
> +clear_tree_failed:
> +userfaultfd_error:
> +munmap_gather_failed:
> +end_split_failed:
> +	mas_set(&mas_detach, 0);
> +	mas_for_each(&mas_detach, next, end)
> +		vma_mark_detached(next, false);
> +
> +	__mt_destroy(&mt_detach);
> +start_split_failed:
> +map_count_exceeded:
> +	validate_mm(mm);
> +	return error;
> +}
> +
> +/*
> + * do_vmi_munmap() - munmap a given range.
> + * @vmi: The vma iterator
> + * @mm: The mm_struct
> + * @start: The start address to munmap
> + * @len: The length of the range to munmap
> + * @uf: The userfaultfd list_head
> + * @unlock: set to true if the user wants to drop the mmap_lock on success
> + *
> + * This function takes a @mas that is either pointing to the previous VMA or set
> + * to MA_START and sets it up to remove the mapping(s).  The @len will be
> + * aligned and any arch_unmap work will be preformed.
> + *
> + * Return: 0 on success and drops the lock if so directed, error and leaves the
> + * lock held otherwise.
> + */
> +int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
> +		  unsigned long start, size_t len, struct list_head *uf,
> +		  bool unlock)
> +{
> +	unsigned long end;
> +	struct vm_area_struct *vma;
> +
> +	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
> +		return -EINVAL;
> +
> +	end = start + PAGE_ALIGN(len);
> +	if (end == start)
> +		return -EINVAL;
> +
> +	/*
> +	 * Check if memory is sealed before arch_unmap.
> +	 * Prevent unmapping a sealed VMA.
> +	 * can_modify_mm assumes we have acquired the lock on MM.
> +	 */
> +	if (unlikely(!can_modify_mm(mm, start, end)))
> +		return -EPERM;
> +
> +	 /* arch_unmap() might do unmaps itself.  */
> +	arch_unmap(mm, start, end);
> +
> +	/* Find the first overlapping VMA */
> +	vma = vma_find(vmi, end);
> +	if (!vma) {
> +		if (unlock)
> +			mmap_write_unlock(mm);
> +		return 0;
> +	}
> +
> +	return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock);
> +}
> +
> +/*
> + * Given a mapping request (addr,end,vm_flags,file,pgoff,anon_name),
> + * figure out whether that can be merged with its predecessor or its
> + * successor.  Or both (it neatly fills a hole).
> + *
> + * In most cases - when called for mmap, brk or mremap - [addr,end) is
> + * certain not to be mapped by the time vma_merge is called; but when
> + * called for mprotect, it is certain to be already mapped (either at
> + * an offset within prev, or at the start of next), and the flags of
> + * this area are about to be changed to vm_flags - and the no-change
> + * case has already been eliminated.
> + *
> + * The following mprotect cases have to be considered, where **** is
> + * the area passed down from mprotect_fixup, never extending beyond one
> + * vma, PPPP is the previous vma, CCCC is a concurrent vma that starts
> + * at the same address as **** and is of the same or larger span, and
> + * NNNN the next vma after ****:
> + *
> + *     ****             ****                   ****
> + *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPCCCCCC
> + *    cannot merge    might become       might become
> + *                    PPNNNNNNNNNN       PPPPPPPPPPCC
> + *    mmap, brk or    case 4 below       case 5 below
> + *    mremap move:
> + *                        ****               ****
> + *                    PPPP    NNNN       PPPPCCCCNNNN
> + *                    might become       might become
> + *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
> + *                    PPPPPPPPNNNN 2 or  PPPPPPPPNNNN 7 or
> + *                    PPPPNNNNNNNN 3     PPPPNNNNNNNN 8
> + *
> + * It is important for case 8 that the vma CCCC overlapping the
> + * region **** is never going to extended over NNNN. Instead NNNN must
> + * be extended in region **** and CCCC must be removed. This way in
> + * all cases where vma_merge succeeds, the moment vma_merge drops the
> + * rmap_locks, the properties of the merged vma will be already
> + * correct for the whole merged range. Some of those properties like
> + * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
> + * be correct for the whole merged range immediately after the
> + * rmap_locks are released. Otherwise if NNNN would be removed and
> + * CCCC would be extended over the NNNN range, remove_migration_ptes
> + * or other rmap walkers (if working on addresses beyond the "end"
> + * parameter) may establish ptes with the wrong permissions of CCCC
> + * instead of the right permissions of NNNN.
> + *
> + * In the code below:
> + * PPPP is represented by *prev
> + * CCCC is represented by *curr or not represented at all (NULL)
> + * NNNN is represented by *next or not represented at all (NULL)
> + * **** is not represented - it will be merged and the vma containing the
> + *      area is returned, or the function will return NULL
> + */
> +static struct vm_area_struct
> +*vma_merge(struct vma_iterator *vmi, struct vm_area_struct *prev,
> +	   struct vm_area_struct *src, unsigned long addr, unsigned long end,
> +	   unsigned long vm_flags, pgoff_t pgoff, struct mempolicy *policy,
> +	   struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> +	   struct anon_vma_name *anon_name)
> +{
> +	struct mm_struct *mm = src->vm_mm;
> +	struct anon_vma *anon_vma = src->anon_vma;
> +	struct file *file = src->vm_file;
> +	struct vm_area_struct *curr, *next, *res;
> +	struct vm_area_struct *vma, *adjust, *remove, *remove2;
> +	struct vm_area_struct *anon_dup = NULL;
> +	struct vma_prepare vp;
> +	pgoff_t vma_pgoff;
> +	int err = 0;
> +	bool merge_prev = false;
> +	bool merge_next = false;
> +	bool vma_expanded = false;
> +	unsigned long vma_start = addr;
> +	unsigned long vma_end = end;
> +	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
> +	long adj_start = 0;
> +
> +	/*
> +	 * We later require that vma->vm_flags == vm_flags,
> +	 * so this tests vma->vm_flags & VM_SPECIAL, too.
> +	 */
> +	if (vm_flags & VM_SPECIAL)
> +		return NULL;
> +
> +	/* Does the input range span an existing VMA? (cases 5 - 8) */
> +	curr = find_vma_intersection(mm, prev ? prev->vm_end : 0, end);
> +
> +	if (!curr ||			/* cases 1 - 4 */
> +	    end == curr->vm_end)	/* cases 6 - 8, adjacent VMA */
> +		next = vma_lookup(mm, end);
> +	else
> +		next = NULL;		/* case 5 */
> +
> +	if (prev) {
> +		vma_start = prev->vm_start;
> +		vma_pgoff = prev->vm_pgoff;
> +
> +		/* Can we merge the predecessor? */
> +		if (addr == prev->vm_end && mpol_equal(vma_policy(prev), policy)
> +		    && can_vma_merge_after(prev, vm_flags, anon_vma, file,
> +					   pgoff, vm_userfaultfd_ctx, anon_name)) {
> +			merge_prev = true;
> +			vma_prev(vmi);
> +		}
> +	}
> +
> +	/* Can we merge the successor? */
> +	if (next && mpol_equal(policy, vma_policy(next)) &&
> +	    can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen,
> +				 vm_userfaultfd_ctx, anon_name)) {
> +		merge_next = true;
> +	}
> +
> +	/* Verify some invariant that must be enforced by the caller. */
> +	VM_WARN_ON(prev && addr <= prev->vm_start);
> +	VM_WARN_ON(curr && (addr != curr->vm_start || end > curr->vm_end));
> +	VM_WARN_ON(addr >= end);
> +
> +	if (!merge_prev && !merge_next)
> +		return NULL; /* Not mergeable. */
> +
> +	if (merge_prev)
> +		vma_start_write(prev);
> +
> +	res = vma = prev;
> +	remove = remove2 = adjust = NULL;
> +
> +	/* Can we merge both the predecessor and the successor? */
> +	if (merge_prev && merge_next &&
> +	    is_mergeable_anon_vma(prev->anon_vma, next->anon_vma, NULL)) {
> +		vma_start_write(next);
> +		remove = next;				/* case 1 */
> +		vma_end = next->vm_end;
> +		err = dup_anon_vma(prev, next, &anon_dup);
> +		if (curr) {				/* case 6 */
> +			vma_start_write(curr);
> +			remove = curr;
> +			remove2 = next;
> +			/*
> +			 * Note that the dup_anon_vma below cannot overwrite err
> +			 * since the first caller would do nothing unless next
> +			 * has an anon_vma.
> +			 */
> +			if (!next->anon_vma)
> +				err = dup_anon_vma(prev, curr, &anon_dup);
> +		}
> +	} else if (merge_prev) {			/* case 2 */
> +		if (curr) {
> +			vma_start_write(curr);
> +			if (end == curr->vm_end) {	/* case 7 */
> +				/*
> +				 * can_vma_merge_after() assumed we would not be
> +				 * removing prev vma, so it skipped the check
> +				 * for vm_ops->close, but we are removing curr
> +				 */
> +				if (curr->vm_ops && curr->vm_ops->close)
> +					err = -EINVAL;
> +				remove = curr;
> +			} else {			/* case 5 */
> +				adjust = curr;
> +				adj_start = (end - curr->vm_start);
> +			}
> +			if (!err)
> +				err = dup_anon_vma(prev, curr, &anon_dup);
> +		}
> +	} else { /* merge_next */
> +		vma_start_write(next);
> +		res = next;
> +		if (prev && addr < prev->vm_end) {	/* case 4 */
> +			vma_start_write(prev);
> +			vma_end = addr;
> +			adjust = next;
> +			adj_start = -(prev->vm_end - addr);
> +			err = dup_anon_vma(next, prev, &anon_dup);
> +		} else {
> +			/*
> +			 * Note that cases 3 and 8 are the ONLY ones where prev
> +			 * is permitted to be (but is not necessarily) NULL.
> +			 */
> +			vma = next;			/* case 3 */
> +			vma_start = addr;
> +			vma_end = next->vm_end;
> +			vma_pgoff = next->vm_pgoff - pglen;
> +			if (curr) {			/* case 8 */
> +				vma_pgoff = curr->vm_pgoff;
> +				vma_start_write(curr);
> +				remove = curr;
> +				err = dup_anon_vma(next, curr, &anon_dup);
> +			}
> +		}
> +	}
> +
> +	/* Error in anon_vma clone. */
> +	if (err)
> +		goto anon_vma_fail;
> +
> +	if (vma_start < vma->vm_start || vma_end > vma->vm_end)
> +		vma_expanded = true;
> +
> +	if (vma_expanded) {
> +		vma_iter_config(vmi, vma_start, vma_end);
> +	} else {
> +		vma_iter_config(vmi, adjust->vm_start + adj_start,
> +				adjust->vm_end);
> +	}
> +
> +	if (vma_iter_prealloc(vmi, vma))
> +		goto prealloc_fail;
> +
> +	init_multi_vma_prep(&vp, vma, adjust, remove, remove2);
> +	VM_WARN_ON(vp.anon_vma && adjust && adjust->anon_vma &&
> +		   vp.anon_vma != adjust->anon_vma);
> +
> +	vma_prepare(&vp);
> +	vma_adjust_trans_huge(vma, vma_start, vma_end, adj_start);
> +	vma_set_range(vma, vma_start, vma_end, vma_pgoff);
> +
> +	if (vma_expanded)
> +		vma_iter_store(vmi, vma);
> +
> +	if (adj_start) {
> +		adjust->vm_start += adj_start;
> +		adjust->vm_pgoff += adj_start >> PAGE_SHIFT;
> +		if (adj_start < 0) {
> +			WARN_ON(vma_expanded);
> +			vma_iter_store(vmi, next);
> +		}
> +	}
> +
> +	vma_complete(&vp, vmi, mm);
> +	khugepaged_enter_vma(res, vm_flags);
> +	return res;
> +
> +prealloc_fail:
> +	if (anon_dup)
> +		unlink_anon_vmas(anon_dup);
> +
> +anon_vma_fail:
> +	vma_iter_set(vmi, addr);
> +	vma_iter_load(vmi);
> +	return NULL;
> +}
> +
> +/*
> + * We are about to modify one or multiple of a VMA's flags, policy, userfaultfd
> + * context and anonymous VMA name within the range [start, end).
> + *
> + * As a result, we might be able to merge the newly modified VMA range with an
> + * adjacent VMA with identical properties.
> + *
> + * If no merge is possible and the range does not span the entirety of the VMA,
> + * we then need to split the VMA to accommodate the change.
> + *
> + * The function returns either the merged VMA, the original VMA if a split was
> + * required instead, or an error if the split failed.
> + */
> +struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> +				  struct vm_area_struct *prev,
> +				  struct vm_area_struct *vma,
> +				  unsigned long start, unsigned long end,
> +				  unsigned long vm_flags,
> +				  struct mempolicy *policy,
> +				  struct vm_userfaultfd_ctx uffd_ctx,
> +				  struct anon_vma_name *anon_name)
> +{
> +	pgoff_t pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> +	struct vm_area_struct *merged;
> +
> +	merged = vma_merge(vmi, prev, vma, start, end, vm_flags,
> +			   pgoff, policy, uffd_ctx, anon_name);
> +	if (merged)
> +		return merged;
> +
> +	if (vma->vm_start < start) {
> +		int err = split_vma(vmi, vma, start, 1);
> +
> +		if (err)
> +			return ERR_PTR(err);
> +	}
> +
> +	if (vma->vm_end > end) {
> +		int err = split_vma(vmi, vma, end, 0);
> +
> +		if (err)
> +			return ERR_PTR(err);
> +	}
> +
> +	return vma;
> +}
> +
> +/*
> + * Attempt to merge a newly mapped VMA with those adjacent to it. The caller
> + * must ensure that [start, end) does not overlap any existing VMA.
> + */
> +struct vm_area_struct
> +*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
> +		   struct vm_area_struct *vma, unsigned long start,
> +		   unsigned long end, pgoff_t pgoff)
> +{
> +	return vma_merge(vmi, prev, vma, start, end, vma->vm_flags, pgoff,
> +			 vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> +}
> +
> +/*
> + * Expand vma by delta bytes, potentially merging with an immediately adjacent
> + * VMA with identical properties.
> + */
> +struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> +					struct vm_area_struct *vma,
> +					unsigned long delta)
> +{
> +	pgoff_t pgoff = vma->vm_pgoff + vma_pages(vma);
> +
> +	/* vma is specified as prev, so case 1 or 2 will apply. */
> +	return vma_merge(vmi, vma, vma, vma->vm_end, vma->vm_end + delta,
> +			 vma->vm_flags, pgoff, vma_policy(vma),
> +			 vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> +}
> +
> +void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb)
> +{
> +	vb->count = 0;
> +}
> +
> +static void unlink_file_vma_batch_process(struct unlink_vma_file_batch *vb)
> +{
> +	struct address_space *mapping;
> +	int i;
> +
> +	mapping = vb->vmas[0]->vm_file->f_mapping;
> +	i_mmap_lock_write(mapping);
> +	for (i = 0; i < vb->count; i++) {
> +		VM_WARN_ON_ONCE(vb->vmas[i]->vm_file->f_mapping != mapping);
> +		__remove_shared_vm_struct(vb->vmas[i], mapping);
> +	}
> +	i_mmap_unlock_write(mapping);
> +
> +	unlink_file_vma_batch_init(vb);
> +}
> +
> +void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
> +			       struct vm_area_struct *vma)
> +{
> +	if (vma->vm_file == NULL)
> +		return;
> +
> +	if ((vb->count > 0 && vb->vmas[0]->vm_file != vma->vm_file) ||
> +	    vb->count == ARRAY_SIZE(vb->vmas))
> +		unlink_file_vma_batch_process(vb);
> +
> +	vb->vmas[vb->count] = vma;
> +	vb->count++;
> +}
> +
> +void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb)
> +{
> +	if (vb->count > 0)
> +		unlink_file_vma_batch_process(vb);
> +}
> +
> +/*
> + * Unlink a file-based vm structure from its interval tree, to hide
> + * vma from rmap and vmtruncate before freeing its page tables.
> + */
> +void unlink_file_vma(struct vm_area_struct *vma)
> +{
> +	struct file *file = vma->vm_file;
> +
> +	if (file) {
> +		struct address_space *mapping = file->f_mapping;
> +
> +		i_mmap_lock_write(mapping);
> +		__remove_shared_vm_struct(vma, mapping);
> +		i_mmap_unlock_write(mapping);
> +	}
> +}
> +
> +void vma_link_file(struct vm_area_struct *vma)
> +{
> +	struct file *file = vma->vm_file;
> +	struct address_space *mapping;
> +
> +	if (file) {
> +		mapping = file->f_mapping;
> +		i_mmap_lock_write(mapping);
> +		__vma_link_file(vma, mapping);
> +		i_mmap_unlock_write(mapping);
> +	}
> +}
> +
> +int vma_link(struct mm_struct *mm, struct vm_area_struct *vma)
> +{
> +	VMA_ITERATOR(vmi, mm, 0);
> +
> +	vma_iter_config(&vmi, vma->vm_start, vma->vm_end);
> +	if (vma_iter_prealloc(&vmi, vma))
> +		return -ENOMEM;
> +
> +	vma_start_write(vma);
> +	vma_iter_store(&vmi, vma);
> +	vma_link_file(vma);
> +	mm->map_count++;
> +	validate_mm(mm);
> +	return 0;
> +}
> +
> +/*
> + * Copy the vma structure to a new location in the same mm,
> + * prior to moving page table entries, to effect an mremap move.
> + */
> +struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> +	unsigned long addr, unsigned long len, pgoff_t pgoff,
> +	bool *need_rmap_locks)
> +{
> +	struct vm_area_struct *vma = *vmap;
> +	unsigned long vma_start = vma->vm_start;
> +	struct mm_struct *mm = vma->vm_mm;
> +	struct vm_area_struct *new_vma, *prev;
> +	bool faulted_in_anon_vma = true;
> +	VMA_ITERATOR(vmi, mm, addr);
> +
> +	/*
> +	 * If anonymous vma has not yet been faulted, update new pgoff
> +	 * to match new location, to increase its chance of merging.
> +	 */
> +	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
> +		pgoff = addr >> PAGE_SHIFT;
> +		faulted_in_anon_vma = false;
> +	}
> +
> +	new_vma = find_vma_prev(mm, addr, &prev);
> +	if (new_vma && new_vma->vm_start < addr + len)
> +		return NULL;	/* should never get here */
> +
> +	new_vma = vma_merge_new_vma(&vmi, prev, vma, addr, addr + len, pgoff);
> +	if (new_vma) {
> +		/*
> +		 * Source vma may have been merged into new_vma
> +		 */
> +		if (unlikely(vma_start >= new_vma->vm_start &&
> +			     vma_start < new_vma->vm_end)) {
> +			/*
> +			 * The only way we can get a vma_merge with
> +			 * self during an mremap is if the vma hasn't
> +			 * been faulted in yet and we were allowed to
> +			 * reset the dst vma->vm_pgoff to the
> +			 * destination address of the mremap to allow
> +			 * the merge to happen. mremap must change the
> +			 * vm_pgoff linearity between src and dst vmas
> +			 * (in turn preventing a vma_merge) to be
> +			 * safe. It is only safe to keep the vm_pgoff
> +			 * linear if there are no pages mapped yet.
> +			 */
> +			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
> +			*vmap = vma = new_vma;
> +		}
> +		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
> +	} else {
> +		new_vma = vm_area_dup(vma);
> +		if (!new_vma)
> +			goto out;
> +		vma_set_range(new_vma, addr, addr + len, pgoff);
> +		if (vma_dup_policy(vma, new_vma))
> +			goto out_free_vma;
> +		if (anon_vma_clone(new_vma, vma))
> +			goto out_free_mempol;
> +		if (new_vma->vm_file)
> +			get_file(new_vma->vm_file);
> +		if (new_vma->vm_ops && new_vma->vm_ops->open)
> +			new_vma->vm_ops->open(new_vma);
> +		if (vma_link(mm, new_vma))
> +			goto out_vma_link;
> +		*need_rmap_locks = false;
> +	}
> +	return new_vma;
> +
> +out_vma_link:
> +	if (new_vma->vm_ops && new_vma->vm_ops->close)
> +		new_vma->vm_ops->close(new_vma);
> +
> +	if (new_vma->vm_file)
> +		fput(new_vma->vm_file);
> +
> +	unlink_anon_vmas(new_vma);
> +out_free_mempol:
> +	mpol_put(vma_policy(new_vma));
> +out_free_vma:
> +	vm_area_free(new_vma);
> +out:
> +	return NULL;
> +}
> +
> +/*
> + * Rough compatibility check to quickly see if it's even worth looking
> + * at sharing an anon_vma.
> + *
> + * They need to have the same vm_file, and the flags can only differ
> + * in things that mprotect may change.
> + *
> + * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
> + * we can merge the two vma's. For example, we refuse to merge a vma if
> + * there is a vm_ops->close() function, because that indicates that the
> + * driver is doing some kind of reference counting. But that doesn't
> + * really matter for the anon_vma sharing case.
> + */
> +static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
> +{
> +	return a->vm_end == b->vm_start &&
> +		mpol_equal(vma_policy(a), vma_policy(b)) &&
> +		a->vm_file == b->vm_file &&
> +		!((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_SOFTDIRTY)) &&
> +		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
> +}
> +
> +/*
> + * Do some basic sanity checking to see if we can re-use the anon_vma
> + * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
> + * the same as 'old', the other will be the new one that is trying
> + * to share the anon_vma.
> + *
> + * NOTE! This runs with mmap_lock held for reading, so it is possible that
> + * the anon_vma of 'old' is concurrently in the process of being set up
> + * by another page fault trying to merge _that_. But that's ok: if it
> + * is being set up, that automatically means that it will be a singleton
> + * acceptable for merging, so we can do all of this optimistically. But
> + * we do that READ_ONCE() to make sure that we never re-load the pointer.
> + *
> + * IOW: that the "list_is_singular()" test on the anon_vma_chain only
> + * matters for the 'stable anon_vma' case (ie the thing we want to avoid
> + * is to return an anon_vma that is "complex" due to having gone through
> + * a fork).
> + *
> + * We also make sure that the two vma's are compatible (adjacent,
> + * and with the same memory policies). That's all stable, even with just
> + * a read lock on the mmap_lock.
> + */
> +static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old,
> +					  struct vm_area_struct *a,
> +					  struct vm_area_struct *b)
> +{
> +	if (anon_vma_compatible(a, b)) {
> +		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
> +
> +		if (anon_vma && list_is_singular(&old->anon_vma_chain))
> +			return anon_vma;
> +	}
> +	return NULL;
> +}
> +
> +/*
> + * find_mergeable_anon_vma is used by anon_vma_prepare, to check
> + * neighbouring vmas for a suitable anon_vma, before it goes off
> + * to allocate a new anon_vma.  It checks because a repetitive
> + * sequence of mprotects and faults may otherwise lead to distinct
> + * anon_vmas being allocated, preventing vma merge in subsequent
> + * mprotect.
> + */
> +struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
> +{
> +	struct anon_vma *anon_vma = NULL;
> +	struct vm_area_struct *prev, *next;
> +	VMA_ITERATOR(vmi, vma->vm_mm, vma->vm_end);
> +
> +	/* Try next first. */
> +	next = vma_iter_load(&vmi);
> +	if (next) {
> +		anon_vma = reusable_anon_vma(next, vma, next);
> +		if (anon_vma)
> +			return anon_vma;
> +	}
> +
> +	prev = vma_prev(&vmi);
> +	VM_BUG_ON_VMA(prev != vma, vma);
> +	prev = vma_prev(&vmi);
> +	/* Try prev next. */
> +	if (prev)
> +		anon_vma = reusable_anon_vma(prev, prev, vma);
> +
> +	/*
> +	 * We might reach here with anon_vma == NULL if we can't find
> +	 * any reusable anon_vma.
> +	 * There's no absolute need to look only at touching neighbours:
> +	 * we could search further afield for "compatible" anon_vmas.
> +	 * But it would probably just be a waste of time searching,
> +	 * or lead to too many vmas hanging off the same anon_vma.
> +	 * We're trying to allow mprotect remerging later on,
> +	 * not trying to minimize memory used for anon_vmas.
> +	 */
> +	return anon_vma;
> +}
> +
> +static bool vm_ops_needs_writenotify(const struct vm_operations_struct *vm_ops)
> +{
> +	return vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite);
> +}
> +
> +static bool vma_is_shared_writable(struct vm_area_struct *vma)
> +{
> +	return (vma->vm_flags & (VM_WRITE | VM_SHARED)) ==
> +		(VM_WRITE | VM_SHARED);
> +}
> +
> +static bool vma_fs_can_writeback(struct vm_area_struct *vma)
> +{
> +	/* No managed pages to writeback. */
> +	if (vma->vm_flags & VM_PFNMAP)
> +		return false;
> +
> +	return vma->vm_file && vma->vm_file->f_mapping &&
> +		mapping_can_writeback(vma->vm_file->f_mapping);
> +}
> +
> +/*
> + * Does this VMA require the underlying folios to have their dirty state
> + * tracked?
> + */
> +bool vma_needs_dirty_tracking(struct vm_area_struct *vma)
> +{
> +	/* Only shared, writable VMAs require dirty tracking. */
> +	if (!vma_is_shared_writable(vma))
> +		return false;
> +
> +	/* Does the filesystem need to be notified? */
> +	if (vm_ops_needs_writenotify(vma->vm_ops))
> +		return true;
> +
> +	/*
> +	 * Even if the filesystem doesn't indicate a need for writenotify, if it
> +	 * can writeback, dirty tracking is still required.
> +	 */
> +	return vma_fs_can_writeback(vma);
> +}
> +
> +/*
> + * Some shared mappings will want the pages marked read-only
> + * to track write events. If so, we'll downgrade vm_page_prot
> + * to the private version (using protection_map[] without the
> + * VM_SHARED bit).
> + */
> +bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
> +{
> +	/* If it was private or non-writable, the write bit is already clear */
> +	if (!vma_is_shared_writable(vma))
> +		return false;
> +
> +	/* The backer wishes to know when pages are first written to? */
> +	if (vm_ops_needs_writenotify(vma->vm_ops))
> +		return true;
> +
> +	/* The open routine did something to the protections that pgprot_modify
> +	 * won't preserve? */
> +	if (pgprot_val(vm_page_prot) !=
> +	    pgprot_val(vm_pgprot_modify(vm_page_prot, vma->vm_flags)))
> +		return false;
> +
> +	/*
> +	 * Do we need to track softdirty? hugetlb does not support softdirty
> +	 * tracking yet.
> +	 */
> +	if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma))
> +		return true;
> +
> +	/* Do we need write faults for uffd-wp tracking? */
> +	if (userfaultfd_wp(vma))
> +		return true;
> +
> +	/* Can the mapping track the dirty pages? */
> +	return vma_fs_can_writeback(vma);
> +}
> +
> +unsigned long count_vma_pages_range(struct mm_struct *mm,
> +				    unsigned long addr, unsigned long end)
> +{
> +	VMA_ITERATOR(vmi, mm, addr);
> +	struct vm_area_struct *vma;
> +	unsigned long nr_pages = 0;
> +
> +	for_each_vma_range(vmi, vma, end) {
> +		unsigned long vm_start = max(addr, vma->vm_start);
> +		unsigned long vm_end = min(end, vma->vm_end);
> +
> +		nr_pages += PHYS_PFN(vm_end - vm_start);
> +	}
> +
> +	return nr_pages;
> +}
> +
> +static DEFINE_MUTEX(mm_all_locks_mutex);
> +
> +static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
> +{
> +	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> +		/*
> +		 * The LSB of head.next can't change from under us
> +		 * because we hold the mm_all_locks_mutex.
> +		 */
> +		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
> +		/*
> +		 * We can safely modify head.next after taking the
> +		 * anon_vma->root->rwsem. If some other vma in this mm shares
> +		 * the same anon_vma we won't take it again.
> +		 *
> +		 * No need of atomic instructions here, head.next
> +		 * can't change from under us thanks to the
> +		 * anon_vma->root->rwsem.
> +		 */
> +		if (__test_and_set_bit(0, (unsigned long *)
> +				       &anon_vma->root->rb_root.rb_root.rb_node))
> +			BUG();
> +	}
> +}
> +
> +static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
> +{
> +	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> +		/*
> +		 * AS_MM_ALL_LOCKS can't change from under us because
> +		 * we hold the mm_all_locks_mutex.
> +		 *
> +		 * Operations on ->flags have to be atomic because
> +		 * even if AS_MM_ALL_LOCKS is stable thanks to the
> +		 * mm_all_locks_mutex, there may be other cpus
> +		 * changing other bitflags in parallel to us.
> +		 */
> +		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
> +			BUG();
> +		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_lock);
> +	}
> +}
> +
> +/*
> + * This operation locks against the VM for all pte/vma/mm related
> + * operations that could ever happen on a certain mm. This includes
> + * vmtruncate, try_to_unmap, and all page faults.
> + *
> + * The caller must take the mmap_lock in write mode before calling
> + * mm_take_all_locks(). The caller isn't allowed to release the
> + * mmap_lock until mm_drop_all_locks() returns.
> + *
> + * mmap_lock in write mode is required in order to block all operations
> + * that could modify pagetables and free pages without need of
> + * altering the vma layout. It's also needed in write mode to avoid new
> + * anon_vmas to be associated with existing vmas.
> + *
> + * A single task can't take more than one mm_take_all_locks() in a row
> + * or it would deadlock.
> + *
> + * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
> + * mapping->flags avoid to take the same lock twice, if more than one
> + * vma in this mm is backed by the same anon_vma or address_space.
> + *
> + * We take locks in following order, accordingly to comment at beginning
> + * of mm/rmap.c:
> + *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
> + *     hugetlb mapping);
> + *   - all vmas marked locked
> + *   - all i_mmap_rwsem locks;
> + *   - all anon_vma->rwseml
> + *
> + * We can take all locks within these types randomly because the VM code
> + * doesn't nest them and we protected from parallel mm_take_all_locks() by
> + * mm_all_locks_mutex.
> + *
> + * mm_take_all_locks() and mm_drop_all_locks are expensive operations
> + * that may have to take thousand of locks.
> + *
> + * mm_take_all_locks() can fail if it's interrupted by signals.
> + */
> +int mm_take_all_locks(struct mm_struct *mm)
> +{
> +	struct vm_area_struct *vma;
> +	struct anon_vma_chain *avc;
> +	VMA_ITERATOR(vmi, mm, 0);
> +
> +	mmap_assert_write_locked(mm);
> +
> +	mutex_lock(&mm_all_locks_mutex);
> +
> +	/*
> +	 * vma_start_write() does not have a complement in mm_drop_all_locks()
> +	 * because vma_start_write() is always asymmetrical; it marks a VMA as
> +	 * being written to until mmap_write_unlock() or mmap_write_downgrade()
> +	 * is reached.
> +	 */
> +	for_each_vma(vmi, vma) {
> +		if (signal_pending(current))
> +			goto out_unlock;
> +		vma_start_write(vma);
> +	}
> +
> +	vma_iter_init(&vmi, mm, 0);
> +	for_each_vma(vmi, vma) {
> +		if (signal_pending(current))
> +			goto out_unlock;
> +		if (vma->vm_file && vma->vm_file->f_mapping &&
> +				is_vm_hugetlb_page(vma))
> +			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> +	}
> +
> +	vma_iter_init(&vmi, mm, 0);
> +	for_each_vma(vmi, vma) {
> +		if (signal_pending(current))
> +			goto out_unlock;
> +		if (vma->vm_file && vma->vm_file->f_mapping &&
> +				!is_vm_hugetlb_page(vma))
> +			vm_lock_mapping(mm, vma->vm_file->f_mapping);
> +	}
> +
> +	vma_iter_init(&vmi, mm, 0);
> +	for_each_vma(vmi, vma) {
> +		if (signal_pending(current))
> +			goto out_unlock;
> +		if (vma->anon_vma)
> +			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> +				vm_lock_anon_vma(mm, avc->anon_vma);
> +	}
> +
> +	return 0;
> +
> +out_unlock:
> +	mm_drop_all_locks(mm);
> +	return -EINTR;
> +}
> +
> +static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
> +{
> +	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
> +		/*
> +		 * The LSB of head.next can't change to 0 from under
> +		 * us because we hold the mm_all_locks_mutex.
> +		 *
> +		 * We must however clear the bitflag before unlocking
> +		 * the vma so the users using the anon_vma->rb_root will
> +		 * never see our bitflag.
> +		 *
> +		 * No need of atomic instructions here, head.next
> +		 * can't change from under us until we release the
> +		 * anon_vma->root->rwsem.
> +		 */
> +		if (!__test_and_clear_bit(0, (unsigned long *)
> +					  &anon_vma->root->rb_root.rb_root.rb_node))
> +			BUG();
> +		anon_vma_unlock_write(anon_vma);
> +	}
> +}
> +
> +static void vm_unlock_mapping(struct address_space *mapping)
> +{
> +	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
> +		/*
> +		 * AS_MM_ALL_LOCKS can't change to 0 from under us
> +		 * because we hold the mm_all_locks_mutex.
> +		 */
> +		i_mmap_unlock_write(mapping);
> +		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
> +					&mapping->flags))
> +			BUG();
> +	}
> +}
> +
> +/*
> + * The mmap_lock cannot be released by the caller until
> + * mm_drop_all_locks() returns.
> + */
> +void mm_drop_all_locks(struct mm_struct *mm)
> +{
> +	struct vm_area_struct *vma;
> +	struct anon_vma_chain *avc;
> +	VMA_ITERATOR(vmi, mm, 0);
> +
> +	mmap_assert_write_locked(mm);
> +	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
> +
> +	for_each_vma(vmi, vma) {
> +		if (vma->anon_vma)
> +			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
> +				vm_unlock_anon_vma(avc->anon_vma);
> +		if (vma->vm_file && vma->vm_file->f_mapping)
> +			vm_unlock_mapping(vma->vm_file->f_mapping);
> +	}
> +
> +	mutex_unlock(&mm_all_locks_mutex);
> +}
> diff --git a/mm/vma.h b/mm/vma.h
> new file mode 100644
> index 000000000000..15d82dbb7213
> --- /dev/null
> +++ b/mm/vma.h
> @@ -0,0 +1,356 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * vma.h
> + *
> + * Core VMA manipulation API implemented in vma.c.
> + */
> +#ifndef __MM_VMA_H
> +#define __MM_VMA_H
> +
> +/*
> + * VMA lock generalization
> + */
> +struct vma_prepare {
> +	struct vm_area_struct *vma;
> +	struct vm_area_struct *adj_next;
> +	struct file *file;
> +	struct address_space *mapping;
> +	struct anon_vma *anon_vma;
> +	struct vm_area_struct *insert;
> +	struct vm_area_struct *remove;
> +	struct vm_area_struct *remove2;
> +};
> +
> +struct unlink_vma_file_batch {
> +	int count;
> +	struct vm_area_struct *vmas[8];
> +};
> +
> +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE
> +void validate_mm(struct mm_struct *mm);
> +#else
> +#define validate_mm(mm) do { } while (0)
> +#endif
> +
> +/* Required for expand_downwards(). */
> +void anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma);
> +
> +/* Required for expand_downwards(). */
> +void anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma);
> +
> +/* Required for do_brk_flags(). */
> +void vma_prepare(struct vma_prepare *vp);
> +
> +/* Required for do_brk_flags(). */
> +void init_vma_prep(struct vma_prepare *vp,
> +		   struct vm_area_struct *vma);
> +
> +/* Required for do_brk_flags(). */
> +void vma_complete(struct vma_prepare *vp,
> +		  struct vma_iterator *vmi, struct mm_struct *mm);
> +
> +int vma_expand(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +	       unsigned long start, unsigned long end, pgoff_t pgoff,
> +	       struct vm_area_struct *next);
> +
> +int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +	       unsigned long start, unsigned long end, pgoff_t pgoff);
> +
> +int
> +do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma,
> +		    struct mm_struct *mm, unsigned long start,
> +		    unsigned long end, struct list_head *uf, bool unlock);
> +
> +int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
> +		  unsigned long start, size_t len, struct list_head *uf,
> +		  bool unlock);
> +
> +void remove_vma(struct vm_area_struct *vma, bool unreachable);
> +
> +void unmap_region(struct mm_struct *mm, struct ma_state *mas,
> +		struct vm_area_struct *vma, struct vm_area_struct *prev,
> +		struct vm_area_struct *next, unsigned long start,
> +		unsigned long end, unsigned long tree_end, bool mm_wr_locked);
> +
> +/* Required by mmap_region(). */
> +bool
> +can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
> +		struct anon_vma *anon_vma, struct file *file,
> +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> +		struct anon_vma_name *anon_name);
> +
> +/* Required by mmap_region() and do_brk_flags(). */
> +bool
> +can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
> +		struct anon_vma *anon_vma, struct file *file,
> +		pgoff_t vm_pgoff, struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> +		struct anon_vma_name *anon_name);
> +
> +struct vm_area_struct *vma_modify(struct vma_iterator *vmi,
> +				  struct vm_area_struct *prev,
> +				  struct vm_area_struct *vma,
> +				  unsigned long start, unsigned long end,
> +				  unsigned long vm_flags,
> +				  struct mempolicy *policy,
> +				  struct vm_userfaultfd_ctx uffd_ctx,
> +				  struct anon_vma_name *anon_name);
> +
> +/* We are about to modify the VMA's flags. */
> +static inline struct vm_area_struct
> +*vma_modify_flags(struct vma_iterator *vmi,
> +		  struct vm_area_struct *prev,
> +		  struct vm_area_struct *vma,
> +		  unsigned long start, unsigned long end,
> +		  unsigned long new_flags)
> +{
> +	return vma_modify(vmi, prev, vma, start, end, new_flags,
> +			  vma_policy(vma), vma->vm_userfaultfd_ctx,
> +			  anon_vma_name(vma));
> +}
> +
> +/* We are about to modify the VMA's flags and/or anon_name. */
> +static inline struct vm_area_struct
> +*vma_modify_flags_name(struct vma_iterator *vmi,
> +		       struct vm_area_struct *prev,
> +		       struct vm_area_struct *vma,
> +		       unsigned long start,
> +		       unsigned long end,
> +		       unsigned long new_flags,
> +		       struct anon_vma_name *new_name)
> +{
> +	return vma_modify(vmi, prev, vma, start, end, new_flags,
> +			  vma_policy(vma), vma->vm_userfaultfd_ctx, new_name);
> +}
> +
> +/* We are about to modify the VMA's memory policy. */
> +static inline struct vm_area_struct
> +*vma_modify_policy(struct vma_iterator *vmi,
> +		   struct vm_area_struct *prev,
> +		   struct vm_area_struct *vma,
> +		   unsigned long start, unsigned long end,
> +		   struct mempolicy *new_pol)
> +{
> +	return vma_modify(vmi, prev, vma, start, end, vma->vm_flags,
> +			  new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma));
> +}
> +
> +/* We are about to modify the VMA's flags and/or uffd context. */
> +static inline struct vm_area_struct
> +*vma_modify_flags_uffd(struct vma_iterator *vmi,
> +		       struct vm_area_struct *prev,
> +		       struct vm_area_struct *vma,
> +		       unsigned long start, unsigned long end,
> +		       unsigned long new_flags,
> +		       struct vm_userfaultfd_ctx new_ctx)
> +{
> +	return vma_modify(vmi, prev, vma, start, end, new_flags,
> +			  vma_policy(vma), new_ctx, anon_vma_name(vma));
> +}
> +
> +struct vm_area_struct
> +*vma_merge_new_vma(struct vma_iterator *vmi, struct vm_area_struct *prev,
> +		   struct vm_area_struct *vma, unsigned long start,
> +		   unsigned long end, pgoff_t pgoff);
> +
> +struct vm_area_struct *vma_merge_extend(struct vma_iterator *vmi,
> +					struct vm_area_struct *vma,
> +					unsigned long delta);
> +
> +void unlink_file_vma_batch_init(struct unlink_vma_file_batch *vb);
> +
> +void unlink_file_vma_batch_final(struct unlink_vma_file_batch *vb);
> +
> +void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb,
> +			       struct vm_area_struct *vma);
> +
> +void unlink_file_vma(struct vm_area_struct *vma);
> +
> +void vma_link_file(struct vm_area_struct *vma);
> +
> +int vma_link(struct mm_struct *mm, struct vm_area_struct *vma);
> +
> +struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> +	unsigned long addr, unsigned long len, pgoff_t pgoff,
> +	bool *need_rmap_locks);
> +
> +struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
> +
> +bool vma_needs_dirty_tracking(struct vm_area_struct *vma);
> +bool vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
> +
> +int mm_take_all_locks(struct mm_struct *mm);
> +void mm_drop_all_locks(struct mm_struct *mm);
> +unsigned long count_vma_pages_range(struct mm_struct *mm,
> +				    unsigned long addr, unsigned long end);
> +
> +static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma)
> +{
> +	/*
> +	 * We want to check manually if we can change individual PTEs writable
> +	 * if we can't do that automatically for all PTEs in a mapping. For
> +	 * private mappings, that's always the case when we have write
> +	 * permissions as we properly have to handle COW.
> +	 */
> +	if (vma->vm_flags & VM_SHARED)
> +		return vma_wants_writenotify(vma, vma->vm_page_prot);
> +	return !!(vma->vm_flags & VM_WRITE);
> +}
> +
> +static inline pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
> +{
> +	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
> +}
> +
> +static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
> +						    unsigned long min)
> +{
> +	return mas_prev(&vmi->mas, min);
> +}
> +
> +static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
> +			struct vm_area_struct *vma, gfp_t gfp)
> +{
> +	if (vmi->mas.status != ma_start &&
> +	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> +		vma_iter_invalidate(vmi);
> +
> +	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> +	mas_store_gfp(&vmi->mas, vma, gfp);
> +	if (unlikely(mas_is_err(&vmi->mas)))
> +		return -ENOMEM;
> +
> +	return 0;
> +}
> +
> +
> +/*
> + * These three helpers classifies VMAs for virtual memory accounting.
> + */
> +
> +/*
> + * Executable code area - executable, not writable, not stack
> + */
> +static inline bool is_exec_mapping(vm_flags_t flags)
> +{
> +	return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
> +}
> +
> +/*
> + * Stack area (including shadow stacks)
> + *
> + * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
> + * do_mmap() forbids all other combinations.
> + */
> +static inline bool is_stack_mapping(vm_flags_t flags)
> +{
> +	return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
> +}
> +
> +/*
> + * Data area - private, writable, not stack
> + */
> +static inline bool is_data_mapping(vm_flags_t flags)
> +{
> +	return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
> +}
> +
> +
> +static inline void vma_iter_config(struct vma_iterator *vmi,
> +		unsigned long index, unsigned long last)
> +{
> +	__mas_set_range(&vmi->mas, index, last - 1);
> +}
> +
> +static inline void vma_iter_reset(struct vma_iterator *vmi)
> +{
> +	mas_reset(&vmi->mas);
> +}
> +
> +static inline
> +struct vm_area_struct *vma_iter_prev_range_limit(struct vma_iterator *vmi, unsigned long min)
> +{
> +	return mas_prev_range(&vmi->mas, min);
> +}
> +
> +static inline
> +struct vm_area_struct *vma_iter_next_range_limit(struct vma_iterator *vmi, unsigned long max)
> +{
> +	return mas_next_range(&vmi->mas, max);
> +}
> +
> +static inline int vma_iter_area_lowest(struct vma_iterator *vmi, unsigned long min,
> +				       unsigned long max, unsigned long size)
> +{
> +	return mas_empty_area(&vmi->mas, min, max - 1, size);
> +}
> +
> +static inline int vma_iter_area_highest(struct vma_iterator *vmi, unsigned long min,
> +					unsigned long max, unsigned long size)
> +{
> +	return mas_empty_area_rev(&vmi->mas, min, max - 1, size);
> +}
> +
> +/*
> + * VMA Iterator functions shared between nommu and mmap
> + */
> +static inline int vma_iter_prealloc(struct vma_iterator *vmi,
> +		struct vm_area_struct *vma)
> +{
> +	return mas_preallocate(&vmi->mas, vma, GFP_KERNEL);
> +}
> +
> +static inline void vma_iter_clear(struct vma_iterator *vmi)
> +{
> +	mas_store_prealloc(&vmi->mas, NULL);
> +}
> +
> +static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
> +{
> +	return mas_walk(&vmi->mas);
> +}
> +
> +/* Store a VMA with preallocated memory */
> +static inline void vma_iter_store(struct vma_iterator *vmi,
> +				  struct vm_area_struct *vma)
> +{
> +
> +#if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
> +	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> +			vmi->mas.index > vma->vm_start)) {
> +		pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
> +			vmi->mas.index, vma->vm_start, vma->vm_start,
> +			vma->vm_end, vmi->mas.index, vmi->mas.last);
> +	}
> +	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
> +			vmi->mas.last <  vma->vm_start)) {
> +		pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
> +		       vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
> +		       vmi->mas.index, vmi->mas.last);
> +	}
> +#endif
> +
> +	if (vmi->mas.status != ma_start &&
> +	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
> +		vma_iter_invalidate(vmi);
> +
> +	__mas_set_range(&vmi->mas, vma->vm_start, vma->vm_end - 1);
> +	mas_store_prealloc(&vmi->mas, vma);
> +}
> +
> +static inline unsigned long vma_iter_addr(struct vma_iterator *vmi)
> +{
> +	return vmi->mas.index;
> +}
> +
> +static inline unsigned long vma_iter_end(struct vma_iterator *vmi)
> +{
> +	return vmi->mas.last + 1;
> +}
> +
> +static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
> +				      unsigned long count)
> +{
> +	return mas_expected_entries(&vmi->mas, count);
> +}
> +
> +#endif	/* __MM_VMA_H */
> diff --git a/mm/vma_internal.h b/mm/vma_internal.h
> new file mode 100644
> index 000000000000..51b2010f30c0
> --- /dev/null
> +++ b/mm/vma_internal.h
> @@ -0,0 +1,143 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * vma_internal.h
> + *
> + * Headers required by vma.c, which can be substituted accordingly when testing
> + * VMA functionality.
> + */
> +

This should probably have header guards for the testing side?

> +/* For fundamental mm types and VMA_ITERATOR(),
> + * tlb_gather_mmu(), tlb_finish_mmu().
> + */
> +#include <linux/mm_types.h>
> +
> +/* For mapping_can_writeback(). */
> +#include <linux/backing-dev.h>
> +
> +/*
> + * For test_bit(), test_and_set_bit(), __test_and_set_bit(),
> + * test_and_clear_bit().
> + */
> +#include <linux/bitops.h>
> +
> +/* For WARN_ON(), WARN_ON_ONCE(), BUG_ON(). */
> +#include <linux/bug.h>
> +
> +/* For ERR_PTR(). */
> +#include <linux/err.h>
> +
> +/* For fput(). */
> +#include <linux/file.h>
> +
> +/*
> + * For get_file(), mapping_unmap_writable(), i_mmap_lock_write(),
> + * i_mmap_unlock_write().
> + */
> +#include <linux/fs.h>
> +
> +/* For vma_adjust_trans_huge(). */
> +#include <linux/huge_mm.h>
> +
> +/* For is_vm_hugetlb_page(). */
> +#include <linux/hugetlb_inline.h>
> +
> +/* For might_sleep(). */
> +#include <linux/kernel.h>
> +
> +/* For khugepaged_enter_vma(). */
> +#include <linux/khugepaged.h>
> +
> +/* For maple tree operations. */
> +#include <linux/maple_tree.h>
> +
> +/* For mpol_put(), vma_policy(), vma_dup_policy(), mpol_equal(). */
> +#include <linux/mempolicy.h>
> +
> +/*
> + * For VM flags, update_hiwater_rss(), __vm_area_free(), vm_area_free(),
> + * vm_area_dup(), unmap_vmas(), vma_start_write(), vma_prev(), vma_next(),
> + * vma_iter_free(), vm_area_free(), vm_stat_account(), vma_is_shared_maywrite(),
> + * vma_interval_tree_remove(), anon_vma_interval_tree_remove(),
> + * anon_vma_interval_tree_insert(), vma_assert_write_locked(), for_each_vma(),
> + * vma_mark_detached(), for_each_vma_range(), vma_iter_set(),
> + * vma_iter_prev_range(), vma_iter_clear_gfp(), PAGE_ALIGN(), vma_find(),
> + * find_vma_intersection(), vma_lookup(), vma_is_anonymous(), find_vma_prev().
> + */
> +#include <linux/mm.h>
> +
> +/* For VM_WARN_ON(), VM_WARN_ON_ONCE_MM(), VM_BUG_ON_VMA(). */
> +#include <linux/mmdebug.h>
> +
> +/* For list_is_singular(), list_for_each_entry(). */
> +#include <linux/list.h>
> +
> +/* For anon_vma_name_eq(). */
> +#include <linux/mm_inline.h>
> +
> +/* For vm_unacct_memory(), vma_pages(). */
> +#include <linux/mman.h>
> +
> +/* For mmap_write_unlock(), mmap_write_downgrade(), mmap_assert_write_locked(). */
> +#include <linux/mmap_lock.h>
> +
> +/* For DEFINE_MUTEX(), mutex_lock(), mutex_unlock(). */
> +#include <linux/mutex.h>
> +
> +/* For AS_MM_ALL_LOCKS. */
> +#include <linux/pagemap.h>
> +
> +/* For PHYS_PFN(). */
> +#include <linux/pfn.h>
> +
> +/* For rcu_read_lock(), rcu_read_unlock(). */
> +#include <linux/rcupdate.h>
> +
> +/*
> + * For anon_vma_clone(), anon_vma_lock_write(), anon_vma_clone(),
> + * unlink_anon_vmas(), anon_vma_unlock_write(), anon_vma_merge().
> + */
> +#include <linux/rmap.h>
> +
> +/* For down_write_nest_lock(). */
> +#include <linux/rwsem.h>
> +
> +/* For signal_pending(). */
> +#include <linux/sched/signal.h>
> +
> +/* For lru_add_drain(). */
> +#include <linux/swap.h>
> +
> +/* For uprobe_mmap(), uprobe_munmap(). */
> +#include <linux/uprobes.h>
> +/*
> + * For struct vm_userfaultfd_ctx, is_mergeable_vm_userfaultfd_ctx(),
> + * userfaultfd_unmap_prep, userfaultfd_wp().
> + */
> +#include <linux/userfaultfd_k.h>
> +
> +/* For BUG(). */
> +#include <linux/bug.h>
> +
> +/* For flush_dcache_mmap_lock(), flush_dcache_mmap_unlock(). */
> +#include <linux/cacheflush.h>
> +
> +/* For current. */
> +#include <asm/current.h>
> +
> +/* For PAGE_SHIFT, etc. */
> +#include <asm/page_types.h>
> +
> +/* For pgprot_val(). */
> +#include <asm/pgtable_types.h>
> +
> +/* For struct mmu_gather. */
> +#include <asm/tlb.h>
> +
> +/* For arch_unmap(). */
> +#include <linux/mmu_context.h>
> +
> +/*
> + * For free_pgtables(), vma_set_range(), can_modify_mm(),
> + * vma_soft_dirty_enabled().
> + */
> +#include "internal.h"
> -- 
> 2.45.1
> 




[Index of Archives]     [Linux Ext4 Filesystem]     [Union Filesystem]     [Filesystem Testing]     [Ceph Users]     [Ecryptfs]     [NTFS 3]     [AutoFS]     [Kernel Newbies]     [Share Photos]     [Security]     [Netfilter]     [Bugtraq]     [Yosemite News]     [MIPS Linux]     [ARM Linux]     [Linux Security]     [Linux Cachefs]     [Reiser Filesystem]     [Linux RAID]     [NTFS 3]     [Samba]     [Device Mapper]     [CEPH Development]

  Powered by Linux