The patch titled
Subject: mm: make mm->pinned_vm an atomic64 counter
has been added to the -mm tree. Its filename is
mm-make-mm-pinned_vm-an-atomic64-counter.patch
This patch should soon appear at
http://ozlabs.org/~akpm/mmots/broken-out/mm-make-mm-pinned_vm-an-atomic64-counter.patch
and later at
http://ozlabs.org/~akpm/mmotm/broken-out/mm-make-mm-pinned_vm-an-atomic64-counter.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated
there every 3-4 working days
------------------------------------------------------
From: Davidlohr Bueso <dave@xxxxxxxxxxxx>
Subject: mm: make mm->pinned_vm an atomic64 counter
Patch series "mm: make pinned_vm atomic and simplify users", v3.
This series aims to provide cleanups to users that pin pages (mostly
infiniband) by converting the counter to atomic -- note that Daniel Jordan
also has patches
(http://lkml.kernel.org/r/20181105165558.11698-8-daniel.m.jordan@xxxxxxxxxx)
for the locked_vm counterpart and vfio.
Apart from removing a source of mmap_sem writer, we benefit in that we can
get rid of a lot of code that defers work when the lock cannot be
acquired, as well as drivers avoiding mmap_sem altogether by also
converting gup to gup_fast() and letting the mm handle it. Users that do
the gup_longterm() remain of course under at least reader mmap_sem.
On a similar topic and potential follow up, it would be nice to resurrect
Peter's VM_PINNED idea in that the broken semantics that occurred after
bc3e53f682 ("mm: distinguish between mlocked and pinned pages") are still
present. Also encapsulating internal mm logic via mm[un]pin() instead of
drivers having to know about internals and playing nice with compaction
are all wins.
[1] https://lkml.org/lkml/2018/11/5/854
This patch (of 6):
Taking a sleeping lock to _only_ increment a variable is quite the
overkill, and pretty much all users do this. Furthermore, some drivers
(ie: infiniband and scif) that need pinned semantics can go to quite some
trouble to actually delay via workqueue (un)accounting for pinned pages
when not possible to acquire it.
By making the counter atomic we no longer need to hold the mmap_sem and
can simply some code around it for pinned_vm users. The counter is 64-bit
such that we need not worry about overflows such as rdma user input
controlled from userspace.
Link: http://lkml.kernel.org/r/20190206175920.31082-2-dave@xxxxxxxxxxxx
Signed-off-by: Davidlohr Bueso <dbueso@xxxxxxx>
Reviewed-by: Ira Weiny <ira.weiny@xxxxxxxxx>
Reviewed-by: Christoph Lameter <cl@xxxxxxxxx>
Reviewed-by: Daniel Jordan <daniel.m.jordan@xxxxxxxxxx>
Reviewed-by: Jan Kara <jack@xxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
--- a/drivers/infiniband/core/umem.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/drivers/infiniband/core/umem.c
@@ -166,13 +166,13 @@ struct ib_umem *ib_umem_get(struct ib_ud
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
down_write(&mm->mmap_sem);
- if (check_add_overflow(mm->pinned_vm, npages, &new_pinned) ||
- (new_pinned > lock_limit && !capable(CAP_IPC_LOCK))) {
+ new_pinned = atomic64_read(&mm->pinned_vm) + npages;
+ if (new_pinned > lock_limit && !capable(CAP_IPC_LOCK)) {
up_write(&mm->mmap_sem);
ret = -ENOMEM;
goto out;
}
- mm->pinned_vm = new_pinned;
+ atomic64_set(&mm->pinned_vm, new_pinned);
up_write(&mm->mmap_sem);
cur_base = addr & PAGE_MASK;
@@ -234,7 +234,7 @@ umem_release:
__ib_umem_release(context->device, umem, 0);
vma:
down_write(&mm->mmap_sem);
- mm->pinned_vm -= ib_umem_num_pages(umem);
+ atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm);
up_write(&mm->mmap_sem);
out:
if (vma_list)
@@ -263,7 +263,7 @@ static void ib_umem_release_defer(struct
struct ib_umem *umem = container_of(work, struct ib_umem, work);
down_write(&umem->owning_mm->mmap_sem);
- umem->owning_mm->pinned_vm -= ib_umem_num_pages(umem);
+ atomic64_sub(ib_umem_num_pages(umem), &umem->owning_mm->pinned_vm);
up_write(&umem->owning_mm->mmap_sem);
__ib_umem_release_tail(umem);
@@ -302,7 +302,7 @@ void ib_umem_release(struct ib_umem *ume
} else {
down_write(&umem->owning_mm->mmap_sem);
}
- umem->owning_mm->pinned_vm -= ib_umem_num_pages(umem);
+ atomic64_sub(ib_umem_num_pages(umem), &umem->owning_mm->pinned_vm);
up_write(&umem->owning_mm->mmap_sem);
__ib_umem_release_tail(umem);
--- a/drivers/infiniband/hw/hfi1/user_pages.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/drivers/infiniband/hw/hfi1/user_pages.c
@@ -92,7 +92,7 @@ bool hfi1_can_pin_pages(struct hfi1_devd
size = DIV_ROUND_UP(size, PAGE_SIZE);
down_read(&mm->mmap_sem);
- pinned = mm->pinned_vm;
+ pinned = atomic64_read(&mm->pinned_vm);
up_read(&mm->mmap_sem);
/* First, check the absolute limit against all pinned pages. */
@@ -112,7 +112,7 @@ int hfi1_acquire_user_pages(struct mm_st
return ret;
down_write(&mm->mmap_sem);
- mm->pinned_vm += ret;
+ atomic64_add(ret, &mm->pinned_vm);
up_write(&mm->mmap_sem);
return ret;
@@ -131,7 +131,7 @@ void hfi1_release_user_pages(struct mm_s
if (mm) { /* during close after signal, mm can be NULL */
down_write(&mm->mmap_sem);
- mm->pinned_vm -= npages;
+ atomic64_sub(npages, &mm->pinned_vm);
up_write(&mm->mmap_sem);
}
}
--- a/drivers/infiniband/hw/qib/qib_user_pages.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/drivers/infiniband/hw/qib/qib_user_pages.c
@@ -75,7 +75,7 @@ static int __qib_get_user_pages(unsigned
goto bail_release;
}
- current->mm->pinned_vm += num_pages;
+ atomic64_add(num_pages, ¤t->mm->pinned_vm);
ret = 0;
goto bail;
@@ -156,7 +156,7 @@ void qib_release_user_pages(struct page
__qib_release_user_pages(p, num_pages, 1);
if (current->mm) {
- current->mm->pinned_vm -= num_pages;
+ atomic64_sub(num_pages, ¤t->mm->pinned_vm);
up_write(¤t->mm->mmap_sem);
}
}
--- a/drivers/infiniband/hw/usnic/usnic_uiom.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/drivers/infiniband/hw/usnic/usnic_uiom.c
@@ -129,7 +129,7 @@ static int usnic_uiom_get_pages(unsigned
uiomr->owning_mm = mm = current->mm;
down_write(&mm->mmap_sem);
- locked = npages + current->mm->pinned_vm;
+ locked = npages + atomic64_read(¤t->mm->pinned_vm);
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
@@ -187,7 +187,7 @@ out:
if (ret < 0)
usnic_uiom_put_pages(chunk_list, 0);
else {
- mm->pinned_vm = locked;
+ atomic64_set(&mm->pinned_vm, locked);
mmgrab(uiomr->owning_mm);
}
@@ -441,7 +441,7 @@ static void usnic_uiom_release_defer(str
container_of(work, struct usnic_uiom_reg, work);
down_write(&uiomr->owning_mm->mmap_sem);
- uiomr->owning_mm->pinned_vm -= usnic_uiom_num_pages(uiomr);
+ atomic64_sub(usnic_uiom_num_pages(uiomr), &uiomr->owning_mm->pinned_vm);
up_write(&uiomr->owning_mm->mmap_sem);
__usnic_uiom_release_tail(uiomr);
@@ -469,7 +469,7 @@ void usnic_uiom_reg_release(struct usnic
} else {
down_write(&uiomr->owning_mm->mmap_sem);
}
- uiomr->owning_mm->pinned_vm -= usnic_uiom_num_pages(uiomr);
+ atomic64_sub(usnic_uiom_num_pages(uiomr), &uiomr->owning_mm->pinned_vm);
up_write(&uiomr->owning_mm->mmap_sem);
__usnic_uiom_release_tail(uiomr);
--- a/drivers/misc/mic/scif/scif_rma.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/drivers/misc/mic/scif/scif_rma.c
@@ -285,7 +285,7 @@ __scif_dec_pinned_vm_lock(struct mm_stru
} else {
down_write(&mm->mmap_sem);
}
- mm->pinned_vm -= nr_pages;
+ atomic64_sub(nr_pages, &mm->pinned_vm);
up_write(&mm->mmap_sem);
return 0;
}
@@ -299,7 +299,7 @@ static inline int __scif_check_inc_pinne
return 0;
locked = nr_pages;
- locked += mm->pinned_vm;
+ locked += atomic64_read(&mm->pinned_vm);
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
dev_err(scif_info.mdev.this_device,
@@ -307,7 +307,7 @@ static inline int __scif_check_inc_pinne
locked, lock_limit);
return -ENOMEM;
}
- mm->pinned_vm = locked;
+ atomic64_set(&mm->pinned_vm, locked);
return 0;
}
--- a/fs/proc/task_mmu.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/fs/proc/task_mmu.c
@@ -59,7 +59,7 @@ void task_mem(struct seq_file *m, struct
SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
- SEQ_PUT_DEC(" kB\nVmPin:\t", mm->pinned_vm);
+ SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
--- a/include/linux/mm_types.h~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/include/linux/mm_types.h
@@ -407,7 +407,7 @@ struct mm_struct {
unsigned long total_vm; /* Total pages mapped */
unsigned long locked_vm; /* Pages that have PG_mlocked set */
- unsigned long pinned_vm; /* Refcount permanently increased */
+ atomic64_t pinned_vm; /* Refcount permanently increased */
unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */
unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
unsigned long stack_vm; /* VM_STACK */
--- a/kernel/events/core.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/kernel/events/core.c
@@ -5465,7 +5465,7 @@ static void perf_mmap_close(struct vm_ar
/* now it's safe to free the pages */
atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm);
- vma->vm_mm->pinned_vm -= rb->aux_mmap_locked;
+ atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
/* this has to be the last one */
rb_free_aux(rb);
@@ -5538,7 +5538,7 @@ again:
*/
atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
- vma->vm_mm->pinned_vm -= mmap_locked;
+ atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
free_uid(mmap_user);
out_put:
@@ -5686,7 +5686,7 @@ accounting:
lock_limit = rlimit(RLIMIT_MEMLOCK);
lock_limit >>= PAGE_SHIFT;
- locked = vma->vm_mm->pinned_vm + extra;
+ locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
!capable(CAP_IPC_LOCK)) {
@@ -5727,7 +5727,7 @@ accounting:
unlock:
if (!ret) {
atomic_long_add(user_extra, &user->locked_vm);
- vma->vm_mm->pinned_vm += extra;
+ atomic64_add(extra, &vma->vm_mm->pinned_vm);
atomic_inc(&event->mmap_count);
} else if (rb) {
--- a/kernel/fork.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/kernel/fork.c
@@ -982,7 +982,7 @@ static struct mm_struct *mm_init(struct
mm_pgtables_bytes_init(mm);
mm->map_count = 0;
mm->locked_vm = 0;
- mm->pinned_vm = 0;
+ atomic64_set(&mm->pinned_vm, 0);
memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
spin_lock_init(&mm->page_table_lock);
spin_lock_init(&mm->arg_lock);
--- a/mm/debug.c~mm-make-mm-pinned_vm-an-atomic64-counter
+++ a/mm/debug.c
@@ -135,7 +135,7 @@ void dump_mm(const struct mm_struct *mm)
"mmap_base %lu mmap_legacy_base %lu highest_vm_end %lu\n"
"pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n"
"hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n"
- "pinned_vm %lx data_vm %lx exec_vm %lx stack_vm %lx\n"
+ "pinned_vm %llx data_vm %lx exec_vm %lx stack_vm %lx\n"
"start_code %lx end_code %lx start_data %lx end_data %lx\n"
"start_brk %lx brk %lx start_stack %lx\n"
"arg_start %lx arg_end %lx env_start %lx env_end %lx\n"
@@ -166,7 +166,8 @@ void dump_mm(const struct mm_struct *mm)
mm_pgtables_bytes(mm),
mm->map_count,
mm->hiwater_rss, mm->hiwater_vm, mm->total_vm, mm->locked_vm,
- mm->pinned_vm, mm->data_vm, mm->exec_vm, mm->stack_vm,
+ atomic64_read(&mm->pinned_vm),
+ mm->data_vm, mm->exec_vm, mm->stack_vm,
mm->start_code, mm->end_code, mm->start_data, mm->end_data,
mm->start_brk, mm->brk, mm->start_stack,
mm->arg_start, mm->arg_end, mm->env_start, mm->env_end,
_
Patches currently in -mm which might be from dave@xxxxxxxxxxxx are
mm-make-mm-pinned_vm-an-atomic64-counter.patch
drivers-mic-scif-do-not-use-mmap_sem.patch
drivers-ibqib-optimize-mmap_sem-usage.patch
drivers-ibhfi1-do-not-se-mmap_sem.patch
drivers-ibusnic-reduce-scope-of-mmap_sem.patch
drivers-ibcore-reduce-scope-of-mmap_sem.patch
