The patch titled
Subject: mm/mmap: use split munmap calls for MAP_FIXED
has been added to the -mm mm-unstable branch. Its filename is
mm-mmap-use-split-munmap-calls-for-map_fixed.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/mm-mmap-use-split-munmap-calls-for-map_fixed.patch
This patch will later appear in the mm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via the mm-everything
branch at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there every 2-3 working days
------------------------------------------------------
From: "Liam R. Howlett" <Liam.Howlett@xxxxxxxxxx>
Subject: mm/mmap: use split munmap calls for MAP_FIXED
Date: Tue, 11 Jun 2024 14:02:00 -0400
Use vms_gather_munmap_vmas() and vms_complete_munmap_vmas() to prepare and
execute the unmapping after the new area is written to the maple tree.
Delaying the unmapping avoids RCU readers seeing a gap in the vmas, which
isn't supposed to exist logically.
Gathering the vmas that will be unmapped allows for the accounting work to
be calculated prior to checking if there is enough memory. Using the
number calculated during vms_gather_munmap_vmas() allows code to be
reduced in mmap_region(). This removes the only caller to
count_vma_pages_range(), so the function has been dropped.
This does have the side effect of allowing vmas to be split for unmap and
fail may_expand_vm(), but the number of pages covered will not change.
The validate_mm() call in vma_expand() must be dropped as the MAP_FIXED
munmap may be in progress. There is still a validate_mm() call at the end
of mmap_region(), which calls vma_expand().
Note that do_vmi_munmap() was previously used to munmap, which checked
alignment and overflow. These checks were unnecessary as do_mmap()
already checks these cases, and arch/mips/kernel/vdso.c
arch_setup_additional_pages() uses predefined values that must already
pass these checks.
Link: https://lkml.kernel.org/r/20240611180200.711239-9-Liam.Howlett@xxxxxxxxxx
Signed-off-by: Liam R. Howlett <Liam.Howlett@xxxxxxxxxx>
Cc: Lorenzo Stoakes <lstoakes@xxxxxxxxx>
Cc: Suren Baghdasaryan <surenb@xxxxxxxxxx>
Cc: Vlastimil Babka <vbabka@xxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
mm/mmap.c | 85 +++++++++++++++++++++++++---------------------------
1 file changed, 42 insertions(+), 43 deletions(-)
--- a/mm/mmap.c~mm-mmap-use-split-munmap-calls-for-map_fixed
+++ a/mm/mmap.c
@@ -405,23 +405,6 @@ anon_vma_interval_tree_post_update_vma(s
anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
}
-static unsigned long count_vma_pages_range(struct mm_struct *mm,
- unsigned long addr, unsigned long end)
-{
- VMA_ITERATOR(vmi, mm, addr);
- struct vm_area_struct *vma;
- unsigned long nr_pages = 0;
-
- for_each_vma_range(vmi, vma, end) {
- unsigned long vm_start = max(addr, vma->vm_start);
- unsigned long vm_end = min(end, vma->vm_end);
-
- nr_pages += PHYS_PFN(vm_end - vm_start);
- }
-
- return nr_pages;
-}
-
static void __vma_link_file(struct vm_area_struct *vma,
struct address_space *mapping)
{
@@ -733,7 +716,6 @@ int vma_expand(struct vma_iterator *vmi,
vma_iter_store(vmi, vma);
vma_complete(&vp, vmi, vma->vm_mm);
- validate_mm(vma->vm_mm);
return 0;
nomem:
@@ -2911,47 +2893,61 @@ unsigned long mmap_region(struct file *f
struct vm_area_struct *next, *prev, *merge;
pgoff_t pglen = len >> PAGE_SHIFT;
unsigned long charged = 0;
+ struct vma_munmap_struct vms;
+ struct ma_state mas_detach;
unsigned long end = addr + len;
unsigned long merge_start = addr, merge_end = end;
bool writable_file_mapping = false;
pgoff_t vm_pgoff;
- int error;
+ int error = -ENOMEM;
VMA_ITERATOR(vmi, mm, addr);
- /* Check against address space limit. */
- if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
- unsigned long nr_pages;
+ vma = vma_find(&vmi, end);
+ if (vma) {
+ struct maple_tree mt_detach;
- /*
- * MAP_FIXED may remove pages of mappings that intersects with
- * requested mapping. Account for the pages it would unmap.
- */
- nr_pages = count_vma_pages_range(mm, addr, end);
+ /* Prevent unmapping a sealed VMA. */
+ if (unlikely(!can_modify_mm(mm, addr, end)))
+ return -EPERM;
- if (!may_expand_vm(mm, vm_flags,
- (len >> PAGE_SHIFT) - nr_pages))
+ mt_init_flags(&mt_detach, vmi.mas.tree->ma_flags & MT_FLAGS_LOCK_MASK);
+ mt_on_stack(mt_detach);
+ mas_init(&mas_detach, &mt_detach, 0);
+ /* arch_unmap() might do unmaps itself. */
+ arch_unmap(mm, addr, end);
+ init_vma_munmap(&vms, &vmi, vma, addr, end, uf,
+ /* unlock = */ false);
+ /* Prepare to unmap any existing mapping in the area */
+ if (vms_gather_munmap_vmas(&vms, &mas_detach))
return -ENOMEM;
+ next = vms.next;
+ prev = vms.prev;
+ vma = NULL;
+ vma_iter_prev_range(&vmi);
+ } else {
+ vms.end = 0; /* vms.end == 0 indicates there is no MAP_FIXED */
+ vms.nr_pages = 0;
+ next = vma_next(&vmi);
+ prev = vma_prev(&vmi);
}
- /* Unmap any existing mapping in the area */
- error = do_vmi_munmap(&vmi, mm, addr, len, uf, false);
- if (error == -EPERM)
- return error;
- else if (error)
- return -ENOMEM;
-
/*
- * Private writable mapping: check memory availability
+ * Check against address space limit.
+ * MAP_FIXED may remove pages of mappings that intersects with
+ * requested mapping. Account for the pages it would unmap.
*/
+ if (!may_expand_vm(mm, vm_flags, (len >> PAGE_SHIFT) - vms.nr_pages))
+ goto no_mem;
+
+ /* Private writable mapping: check memory availability */
if (accountable_mapping(file, vm_flags)) {
charged = len >> PAGE_SHIFT;
+ charged -= vms.nr_pages; /* MAP_FIXED removed memory */
if (security_vm_enough_memory_mm(mm, charged))
- return -ENOMEM;
+ goto no_mem;
vm_flags |= VM_ACCOUNT;
}
- next = vma_next(&vmi);
- prev = vma_prev(&vmi);
if (vm_flags & VM_SPECIAL) {
if (prev)
vma_iter_next_range(&vmi);
@@ -2998,10 +2994,8 @@ cannot_expand:
* not unmapped, but the maps are removed from the list.
*/
vma = vm_area_alloc(mm);
- if (!vma) {
- error = -ENOMEM;
+ if (!vma)
goto unacct_error;
- }
vma_iter_config(&vmi, addr, end);
vma_set_range(vma, addr, end, pgoff);
@@ -3123,6 +3117,8 @@ expanded:
vm_flags_set(vma, VM_SOFTDIRTY);
vma_set_page_prot(vma);
+ if (vms.end)
+ vms_complete_munmap_vmas(&vms, &mas_detach);
validate_mm(mm);
return addr;
@@ -3148,6 +3144,9 @@ free_vma:
unacct_error:
if (charged)
vm_unacct_memory(charged);
+no_mem:
+ if (vms.end)
+ abort_munmap_vmas(&mas_detach);
validate_mm(mm);
return error;
}
_
Patches currently in -mm which might be from Liam.Howlett@xxxxxxxxxx are
mm-mmap-correctly-position-vma_iterator-in-__split_vma.patch
mm-mmap-introduce-abort_munmap_vmas.patch
mm-mmap-introduce-vmi_complete_munmap_vmas.patch
mm-mmap-extract-the-gathering-of-vmas-from-do_vmi_align_munmap.patch
mm-mmap-introduce-vma_munmap_struct-for-use-in-munmap-operations.patch
mm-mmap-change-munmap-to-use-vma_munmap_struct-for-accounting-and-surrounding-vmas.patch
mm-mmap-extract-validate_mm-from-vma_complete.patch
mm-mmap-use-split-munmap-calls-for-map_fixed.patch
