The patch titled
Subject: mm: switch vma_merge(), split_vma(), and __split_vma to vma iterator
has been added to the -mm mm-unstable branch. Its filename is
mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator.patch
This patch will later appear in the mm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via the mm-everything
branch at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there every 2-3 working days
------------------------------------------------------
From: "Liam R. Howlett" <Liam.Howlett@xxxxxxxxxx>
Subject: mm: switch vma_merge(), split_vma(), and __split_vma to vma iterator
Date: Fri, 20 Jan 2023 11:26:30 -0500
Drop the vmi_* functions and transition all users to use the vma iterator
directly.
Link: https://lkml.kernel.org/r/20230120162650.984577-30-Liam.Howlett@xxxxxxxxxx
Signed-off-by: Liam R. Howlett <Liam.Howlett@xxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
--- a/fs/userfaultfd.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/fs/userfaultfd.c
@@ -909,7 +909,7 @@ static int userfaultfd_release(struct in
continue;
}
new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
- prev = vmi_vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
+ prev = vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
new_flags, vma->anon_vma,
vma->vm_file, vma->vm_pgoff,
vma_policy(vma),
@@ -1452,7 +1452,7 @@ static int userfaultfd_register(struct u
vma_end = min(end, vma->vm_end);
new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
- prev = vmi_vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
+ prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
vma->anon_vma, vma->vm_file, vma->vm_pgoff,
vma_policy(vma),
((struct vm_userfaultfd_ctx){ ctx }),
@@ -1463,12 +1463,12 @@ static int userfaultfd_register(struct u
goto next;
}
if (vma->vm_start < start) {
- ret = vmi_split_vma(&vmi, mm, vma, start, 1);
+ ret = split_vma(&vmi, vma, start, 1);
if (ret)
break;
}
if (vma->vm_end > end) {
- ret = vmi_split_vma(&vmi, mm, vma, end, 0);
+ ret = split_vma(&vmi, vma, end, 0);
if (ret)
break;
}
@@ -1632,7 +1632,7 @@ static int userfaultfd_unregister(struct
uffd_wp_range(mm, vma, start, vma_end - start, false);
new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
- prev = vmi_vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
+ prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
vma->anon_vma, vma->vm_file, vma->vm_pgoff,
vma_policy(vma),
NULL_VM_UFFD_CTX, anon_vma_name(vma));
@@ -1641,12 +1641,12 @@ static int userfaultfd_unregister(struct
goto next;
}
if (vma->vm_start < start) {
- ret = vmi_split_vma(&vmi, mm, vma, start, 1);
+ ret = split_vma(&vmi, vma, start, 1);
if (ret)
break;
}
if (vma->vm_end > end) {
- ret = vmi_split_vma(&vmi, mm, vma, end, 0);
+ ret = split_vma(&vmi, vma, end, 0);
if (ret)
break;
}
--- a/include/linux/mm.h~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/include/linux/mm.h
@@ -2839,24 +2839,16 @@ static inline int vma_adjust(struct vm_a
{
return __vma_adjust(vma, start, end, pgoff, insert, NULL);
}
-extern struct vm_area_struct *vma_merge(struct mm_struct *,
- struct vm_area_struct *prev, unsigned long addr, unsigned long end,
- unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
- struct mempolicy *, struct vm_userfaultfd_ctx, struct anon_vma_name *);
-extern struct vm_area_struct *vmi_vma_merge(struct vma_iterator *vmi,
+extern struct vm_area_struct *vma_merge(struct vma_iterator *vmi,
struct mm_struct *, struct vm_area_struct *prev, unsigned long addr,
unsigned long end, unsigned long vm_flags, struct anon_vma *,
struct file *, pgoff_t, struct mempolicy *, struct vm_userfaultfd_ctx,
struct anon_vma_name *);
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
-extern int __split_vma(struct mm_struct *, struct vm_area_struct *,
- unsigned long addr, int new_below);
-extern int vmi__split_vma(struct vma_iterator *vmi, struct mm_struct *,
- struct vm_area_struct *, unsigned long addr, int new_below);
-extern int split_vma(struct mm_struct *, struct vm_area_struct *,
- unsigned long addr, int new_below);
-extern int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *,
- struct vm_area_struct *, unsigned long addr, int new_below);
+extern int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *,
+ unsigned long addr, int new_below);
+extern int split_vma(struct vma_iterator *vmi, struct vm_area_struct *,
+ unsigned long addr, int new_below);
extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
extern void unlink_file_vma(struct vm_area_struct *);
extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
--- a/mm/madvise.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/madvise.c
@@ -150,7 +150,7 @@ static int madvise_update_vma(struct vm_
}
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
- *prev = vmi_vma_merge(&vmi, mm, *prev, start, end, new_flags,
+ *prev = vma_merge(&vmi, mm, *prev, start, end, new_flags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_name);
if (*prev) {
@@ -163,7 +163,7 @@ static int madvise_update_vma(struct vm_
if (start != vma->vm_start) {
if (unlikely(mm->map_count >= sysctl_max_map_count))
return -ENOMEM;
- error = vmi__split_vma(&vmi, mm, vma, start, 1);
+ error = __split_vma(&vmi, vma, start, 1);
if (error)
return error;
}
@@ -171,7 +171,7 @@ static int madvise_update_vma(struct vm_
if (end != vma->vm_end) {
if (unlikely(mm->map_count >= sysctl_max_map_count))
return -ENOMEM;
- error = vmi__split_vma(&vmi, mm, vma, end, 0);
+ error = __split_vma(&vmi, vma, end, 0);
if (error)
return error;
}
--- a/mm/mempolicy.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/mempolicy.c
@@ -809,7 +809,7 @@ static int mbind_range(struct mm_struct
pgoff = vma->vm_pgoff +
((vmstart - vma->vm_start) >> PAGE_SHIFT);
- prev = vmi_vma_merge(&vmi, mm, prev, vmstart, vmend, vma->vm_flags,
+ prev = vma_merge(&vmi, mm, prev, vmstart, vmend, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff,
new_pol, vma->vm_userfaultfd_ctx,
anon_vma_name(vma));
@@ -818,12 +818,12 @@ static int mbind_range(struct mm_struct
goto replace;
}
if (vma->vm_start != vmstart) {
- err = vmi_split_vma(&vmi, vma->vm_mm, vma, vmstart, 1);
+ err = split_vma(&vmi, vma, vmstart, 1);
if (err)
goto out;
}
if (vma->vm_end != vmend) {
- err = vmi_split_vma(&vmi, vma->vm_mm, vma, vmend, 0);
+ err = split_vma(&vmi, vma, vmend, 0);
if (err)
goto out;
}
--- a/mm/mlock.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/mlock.c
@@ -418,7 +418,7 @@ static int mlock_fixup(struct vma_iterat
goto out;
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
- *prev = vmi_vma_merge(vmi, mm, *prev, start, end, newflags,
+ *prev = vma_merge(vmi, mm, *prev, start, end, newflags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_vma_name(vma));
if (*prev) {
@@ -427,13 +427,13 @@ static int mlock_fixup(struct vma_iterat
}
if (start != vma->vm_start) {
- ret = vmi_split_vma(vmi, mm, vma, start, 1);
+ ret = split_vma(vmi, vma, start, 1);
if (ret)
goto out;
}
if (end != vma->vm_end) {
- ret = vmi_split_vma(vmi, mm, vma, end, 0);
+ ret = split_vma(vmi, vma, end, 0);
if (ret)
goto out;
}
--- a/mm/mmap.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/mmap.c
@@ -1013,7 +1013,7 @@ can_vma_merge_after(struct vm_area_struc
* parameter) may establish ptes with the wrong permissions of NNNN
* instead of the right permissions of XXXX.
*/
-struct vm_area_struct *vma_merge(struct mm_struct *mm,
+struct vm_area_struct *vma_merge(struct vma_iterator *vmi, struct mm_struct *mm,
struct vm_area_struct *prev, unsigned long addr,
unsigned long end, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file,
@@ -1022,7 +1022,7 @@ struct vm_area_struct *vma_merge(struct
struct anon_vma_name *anon_name)
{
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
- struct vm_area_struct *mid, *next, *res;
+ struct vm_area_struct *mid, *next, *res = NULL;
int err = -1;
bool merge_prev = false;
bool merge_next = false;
@@ -1088,26 +1088,11 @@ struct vm_area_struct *vma_merge(struct
if (err)
return NULL;
khugepaged_enter_vma(res, vm_flags);
- return res;
-}
-struct vm_area_struct *vmi_vma_merge(struct vma_iterator *vmi,
- struct mm_struct *mm,
- struct vm_area_struct *prev, unsigned long addr,
- unsigned long end, unsigned long vm_flags,
- struct anon_vma *anon_vma, struct file *file,
- pgoff_t pgoff, struct mempolicy *policy,
- struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
- struct anon_vma_name *anon_name)
-{
- struct vm_area_struct *tmp;
-
- tmp = vma_merge(mm, prev, addr, end, vm_flags, anon_vma, file, pgoff,
- policy, vm_userfaultfd_ctx, anon_name);
- if (tmp)
+ if (res)
vma_iter_set(vmi, end);
- return tmp;
+ return res;
}
/*
@@ -2231,12 +2216,14 @@ static void unmap_region(struct mm_struc
* __split_vma() bypasses sysctl_max_map_count checking. We use this where it
* has already been checked or doesn't make sense to fail.
*/
-int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+int __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long addr, int new_below)
{
struct vm_area_struct *new;
int err;
- validate_mm_mt(mm);
+ unsigned long end = vma->vm_end;
+
+ validate_mm_mt(vma->vm_mm);
if (vma->vm_ops && vma->vm_ops->may_split) {
err = vma->vm_ops->may_split(vma, addr);
@@ -2276,8 +2263,10 @@ int __split_vma(struct mm_struct *mm, st
err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
/* Success. */
- if (!err)
+ if (!err) {
+ vma_iter_set(vmi, end);
return 0;
+ }
/* Avoid vm accounting in close() operation */
new->vm_start = new->vm_end;
@@ -2292,46 +2281,21 @@ int __split_vma(struct mm_struct *mm, st
mpol_put(vma_policy(new));
out_free_vma:
vm_area_free(new);
- validate_mm_mt(mm);
+ validate_mm_mt(vma->vm_mm);
return err;
}
-int vmi__split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
- struct vm_area_struct *vma, unsigned long addr, int new_below)
-{
- int ret;
- unsigned long end = vma->vm_end;
-
- ret = __split_vma(mm, vma, addr, new_below);
- if (!ret)
- vma_iter_set(vmi, end);
-
- return ret;
-}
/*
* Split a vma into two pieces at address 'addr', a new vma is allocated
* either for the first part or the tail.
*/
-int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
+int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
unsigned long addr, int new_below)
{
- if (mm->map_count >= sysctl_max_map_count)
+ if (vma->vm_mm->map_count >= sysctl_max_map_count)
return -ENOMEM;
- return __split_vma(mm, vma, addr, new_below);
-}
-
-int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
- struct vm_area_struct *vma, unsigned long addr, int new_below)
-{
- int ret;
- unsigned long end = vma->vm_end;
-
- ret = split_vma(mm, vma, addr, new_below);
- if (!ret)
- vma_iter_set(vmi, end);
-
- return ret;
+ return __split_vma(vmi, vma, addr, new_below);
}
static inline int munmap_sidetree(struct vm_area_struct *vma,
@@ -2391,7 +2355,7 @@ do_vmi_align_munmap(struct vma_iterator
if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
goto map_count_exceeded;
- error = vmi__split_vma(vmi, mm, vma, start, 0);
+ error = __split_vma(vmi, vma, start, 0);
if (error)
goto start_split_failed;
@@ -2412,7 +2376,7 @@ do_vmi_align_munmap(struct vma_iterator
if (next->vm_end > end) {
struct vm_area_struct *split;
- error = vmi__split_vma(vmi, mm, next, end, 1);
+ error = __split_vma(vmi, next, end, 1);
if (error)
goto end_split_failed;
@@ -2693,9 +2657,10 @@ cannot_expand:
* vma again as we may succeed this time.
*/
if (unlikely(vm_flags != vma->vm_flags && prev)) {
- merge = vmi_vma_merge(&vmi, mm, prev, vma->vm_start,
- vma->vm_end, vma->vm_flags, NULL, vma->vm_file,
- vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL);
+ merge = vma_merge(&vmi, mm, prev, vma->vm_start,
+ vma->vm_end, vma->vm_flags, NULL,
+ vma->vm_file, vma->vm_pgoff, NULL,
+ NULL_VM_UFFD_CTX, NULL);
if (merge) {
/*
* ->mmap() can change vma->vm_file and fput
@@ -3251,7 +3216,7 @@ struct vm_area_struct *copy_vma(struct v
if (new_vma && new_vma->vm_start < addr + len)
return NULL; /* should never get here */
- new_vma = vmi_vma_merge(&vmi, mm, prev, addr, addr + len, vma->vm_flags,
+ new_vma = vma_merge(&vmi, mm, prev, addr, addr + len, vma->vm_flags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_vma_name(vma));
if (new_vma) {
--- a/mm/mprotect.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/mprotect.c
@@ -642,7 +642,7 @@ mprotect_fixup(struct vma_iterator *vmi,
* First try to merge with previous and/or next vma.
*/
pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
- *pprev = vmi_vma_merge(vmi, mm, *pprev, start, end, newflags,
+ *pprev = vma_merge(vmi, mm, *pprev, start, end, newflags,
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
vma->vm_userfaultfd_ctx, anon_vma_name(vma));
if (*pprev) {
@@ -654,13 +654,13 @@ mprotect_fixup(struct vma_iterator *vmi,
*pprev = vma;
if (start != vma->vm_start) {
- error = vmi_split_vma(vmi, mm, vma, start, 1);
+ error = split_vma(vmi, vma, start, 1);
if (error)
goto fail;
}
if (end != vma->vm_end) {
- error = vmi_split_vma(vmi, mm, vma, end, 0);
+ error = split_vma(vmi, vma, end, 0);
if (error)
goto fail;
}
--- a/mm/mremap.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/mremap.c
@@ -1043,12 +1043,10 @@ SYSCALL_DEFINE5(mremap, unsigned long, a
* when a vma would be actually removed due to a merge.
*/
if (!vma->vm_ops || !vma->vm_ops->close) {
- vma = vmi_vma_merge(&vmi, mm, vma,
- extension_start, extension_end,
- vma->vm_flags, vma->anon_vma,
- vma->vm_file, extension_pgoff,
- vma_policy(vma), vma->vm_userfaultfd_ctx,
- anon_vma_name(vma));
+ vma = vma_merge(&vmi, mm, vma, extension_start,
+ extension_end, vma->vm_flags, vma->anon_vma,
+ vma->vm_file, extension_pgoff, vma_policy(vma),
+ vma->vm_userfaultfd_ctx, anon_vma_name(vma));
} else if (vma_adjust(vma, vma->vm_start, addr + new_len,
vma->vm_pgoff, NULL)) {
vma = NULL;
--- a/mm/nommu.c~mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator
+++ a/mm/nommu.c
@@ -1297,18 +1297,20 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_ar
* split a vma into two pieces at address 'addr', a new vma is allocated either
* for the first part or the tail.
*/
-int vmi_split_vma(struct vma_iterator *vmi, struct mm_struct *mm,
- struct vm_area_struct *vma, unsigned long addr, int new_below)
+int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma,
+ unsigned long addr, int new_below)
{
struct vm_area_struct *new;
struct vm_region *region;
unsigned long npages;
+ struct mm_struct *mm;
/* we're only permitted to split anonymous regions (these should have
* only a single usage on the region) */
if (vma->vm_file)
return -ENOMEM;
+ mm = vma->vm_mm;
if (mm->map_count >= sysctl_max_map_count)
return -ENOMEM;
@@ -1465,7 +1467,7 @@ int do_munmap(struct mm_struct *mm, unsi
if (end != vma->vm_end && offset_in_page(end))
return -EINVAL;
if (start != vma->vm_start && end != vma->vm_end) {
- ret = vmi_split_vma(&vmi, mm, vma, start, 1);
+ ret = split_vma(&vmi, vma, start, 1);
if (ret < 0)
return ret;
}
_
Patches currently in -mm which might be from Liam.Howlett@xxxxxxxxxx are
maple_tree-add-mas_init-function.patch
maple_tree-fix-potential-rcu-issue.patch
maple_tree-reduce-user-error-potential.patch
test_maple_tree-test-modifications-while-iterating.patch
mm-expand-vma-iterator-interface.patch
mm-mmap-convert-brk-to-use-vma-iterator.patch
kernel-fork-convert-forking-to-using-the-vmi-iterator.patch
mmap-convert-vma_link-vma-iterator.patch
mm-mmap-remove-preallocation-from-do_mas_align_munmap.patch
mmap-change-do_mas_munmap-and-do_mas_aligned_munmap-to-use-vma-iterator.patch
mmap-convert-vma_expand-to-use-vma-iterator.patch
mm-add-temporary-vma-iterator-versions-of-vma_merge-split_vma-and-__split_vma.patch
ipc-shm-use-the-vma-iterator-for-munmap-calls.patch
userfaultfd-use-vma-iterator.patch
mm-change-mprotect_fixup-to-vma-iterator.patch
mlock-convert-mlock-to-vma-iterator.patch
coredump-convert-to-vma-iterator.patch
mempolicy-convert-to-vma-iterator.patch
task_mmu-convert-to-vma-iterator.patch
sched-convert-to-vma-iterator.patch
madvise-use-vmi-iterator-for-__split_vma-and-vma_merge.patch
mmap-pass-through-vmi-iterator-to-__split_vma.patch
mmap-use-vmi-version-of-vma_merge.patch
mm-mremap-use-vmi-version-of-vma_merge.patch
nommu-convert-nommu-to-using-the-vma-iterator.patch
mm-switch-vma_merge-split_vma-and-__split_vma-to-vma-iterator.patch
mmap-convert-__vma_adjust-to-use-vma-iterator.patch
mm-pass-through-vma-iterator-to-__vma_adjust.patch
madvise-use-split_vma-instead-of-__split_vma.patch
mm-remove-unnecessary-write-to-vma-iterator-in-__vma_adjust.patch
mm-pass-vma-iterator-through-to-__vma_adjust.patch
mm-add-vma-iterator-to-vma_adjust-arguments.patch
mmap-clean-up-mmap_region-unrolling.patch
mm-change-munmap-splitting-order-and-move_vma.patch
mm-mmap-move-anon_vma-setting-in-__vma_adjust.patch
mm-mmap-refactor-locking-out-of-__vma_adjust.patch
mm-mmap-use-vma_prepare-and-vma_complete-in-vma_expand.patch
mm-mmap-introduce-init_vma_prep-and-init_multi_vma_prep.patch
mm-dont-use-__vma_adjust-in-__split_vma.patch
mm-mmap-dont-use-__vma_adjust-in-shift_arg_pages.patch
mm-mmap-introduce-dup_vma_anon-helper.patch
mm-mmap-convert-do_brk_flags-to-use-vma_prepare-and-vma_complete.patch
mm-mmap-remove-__vma_adjust.patch
vma_merge-set-vma-iterator-to-correct-position.patch
