Hi Andrea,
On 5 March 2015 at 18:17, Andrea Arcangeli <aarcange@xxxxxxxxxx> wrote:
> Once an userfaultfd has been created and certain region of the process
> virtual address space have been registered into it, the thread
> responsible for doing the memory externalization can manage the page
> faults in userland by talking to the kernel using the userfaultfd
> protocol.
Is there someting like a man page for this new syscall?
Thanks,
Michael
> poll() can be used to know when there are new pending userfaults to be
> read (POLLIN).
>
> Signed-off-by: Andrea Arcangeli <aarcange@xxxxxxxxxx>
> ---
> fs/userfaultfd.c | 977 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 977 insertions(+)
> create mode 100644 fs/userfaultfd.c
>
> diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
> new file mode 100644
> index 0000000..6b31967
> --- /dev/null
> +++ b/fs/userfaultfd.c
> @@ -0,0 +1,977 @@
> +/*
> + * fs/userfaultfd.c
> + *
> + * Copyright (C) 2007 Davide Libenzi <davidel@xxxxxxxxxxxxxxx>
> + * Copyright (C) 2008-2009 Red Hat, Inc.
> + * Copyright (C) 2015 Red Hat, Inc.
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2. See
> + * the COPYING file in the top-level directory.
> + *
> + * Some part derived from fs/eventfd.c (anon inode setup) and
> + * mm/ksm.c (mm hashing).
> + */
> +
> +#include <linux/hashtable.h>
> +#include <linux/sched.h>
> +#include <linux/mm.h>
> +#include <linux/poll.h>
> +#include <linux/slab.h>
> +#include <linux/seq_file.h>
> +#include <linux/file.h>
> +#include <linux/bug.h>
> +#include <linux/anon_inodes.h>
> +#include <linux/syscalls.h>
> +#include <linux/userfaultfd_k.h>
> +#include <linux/mempolicy.h>
> +#include <linux/ioctl.h>
> +#include <linux/security.h>
> +
> +enum userfaultfd_state {
> + UFFD_STATE_WAIT_API,
> + UFFD_STATE_RUNNING,
> +};
> +
> +struct userfaultfd_ctx {
> + /* pseudo fd refcounting */
> + atomic_t refcount;
> + /* waitqueue head for the userfaultfd page faults */
> + wait_queue_head_t fault_wqh;
> + /* waitqueue head for the pseudo fd to wakeup poll/read */
> + wait_queue_head_t fd_wqh;
> + /* userfaultfd syscall flags */
> + unsigned int flags;
> + /* state machine */
> + enum userfaultfd_state state;
> + /* released */
> + bool released;
> + /* mm with one ore more vmas attached to this userfaultfd_ctx */
> + struct mm_struct *mm;
> +};
> +
> +struct userfaultfd_wait_queue {
> + unsigned long address;
> + wait_queue_t wq;
> + bool pending;
> + struct userfaultfd_ctx *ctx;
> +};
> +
> +struct userfaultfd_wake_range {
> + unsigned long start;
> + unsigned long len;
> +};
> +
> +static int userfaultfd_wake_function(wait_queue_t *wq, unsigned mode,
> + int wake_flags, void *key)
> +{
> + struct userfaultfd_wake_range *range = key;
> + int ret;
> + struct userfaultfd_wait_queue *uwq;
> + unsigned long start, len;
> +
> + uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
> + ret = 0;
> + /* don't wake the pending ones to avoid reads to block */
> + if (uwq->pending && !ACCESS_ONCE(uwq->ctx->released))
> + goto out;
> + /* len == 0 means wake all */
> + start = range->start;
> + len = range->len;
> + if (len && (start > uwq->address || start + len <= uwq->address))
> + goto out;
> + ret = wake_up_state(wq->private, mode);
> + if (ret)
> + /* wake only once, autoremove behavior */
> + list_del_init(&wq->task_list);
> +out:
> + return ret;
> +}
> +
> +/**
> + * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
> + * context.
> + * @ctx: [in] Pointer to the userfaultfd context.
> + *
> + * Returns: In case of success, returns not zero.
> + */
> +static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
> +{
> + if (!atomic_inc_not_zero(&ctx->refcount))
> + BUG();
> +}
> +
> +/**
> + * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
> + * context.
> + * @ctx: [in] Pointer to userfaultfd context.
> + *
> + * The userfaultfd context reference must have been previously acquired either
> + * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
> + */
> +static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
> +{
> + if (atomic_dec_and_test(&ctx->refcount)) {
> + mmdrop(ctx->mm);
> + kfree(ctx);
> + }
> +}
> +
> +static inline unsigned long userfault_address(unsigned long address,
> + unsigned int flags,
> + unsigned long reason)
> +{
> + BUILD_BUG_ON(PAGE_SHIFT < UFFD_BITS);
> + address &= PAGE_MASK;
> + if (flags & FAULT_FLAG_WRITE)
> + /*
> + * Encode "write" fault information in the LSB of the
> + * address read by userland, without depending on
> + * FAULT_FLAG_WRITE kernel internal value.
> + */
> + address |= UFFD_BIT_WRITE;
> + if (reason & VM_UFFD_WP)
> + /*
> + * Encode "reason" fault information as bit number 1
> + * in the address read by userland. If bit number 1 is
> + * clear it means the reason is a VM_FAULT_MISSING
> + * fault.
> + */
> + address |= UFFD_BIT_WP;
> + return address;
> +}
> +
> +/*
> + * The locking rules involved in returning VM_FAULT_RETRY depending on
> + * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
> + * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
> + * recommendation in __lock_page_or_retry is not an understatement.
> + *
> + * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released
> + * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
> + * not set.
> + *
> + * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
> + * set, VM_FAULT_RETRY can still be returned if and only if there are
> + * fatal_signal_pending()s, and the mmap_sem must be released before
> + * returning it.
> + */
> +int handle_userfault(struct vm_area_struct *vma, unsigned long address,
> + unsigned int flags, unsigned long reason)
> +{
> + struct mm_struct *mm = vma->vm_mm;
> + struct userfaultfd_ctx *ctx;
> + struct userfaultfd_wait_queue uwq;
> +
> + BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
> +
> + ctx = vma->vm_userfaultfd_ctx.ctx;
> + if (!ctx)
> + return VM_FAULT_SIGBUS;
> +
> + BUG_ON(ctx->mm != mm);
> +
> + VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP));
> + VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP));
> +
> + /*
> + * If it's already released don't get it. This avoids to loop
> + * in __get_user_pages if userfaultfd_release waits on the
> + * caller of handle_userfault to release the mmap_sem.
> + */
> + if (unlikely(ACCESS_ONCE(ctx->released)))
> + return VM_FAULT_SIGBUS;
> +
> + /* check that we can return VM_FAULT_RETRY */
> + if (unlikely(!(flags & FAULT_FLAG_ALLOW_RETRY))) {
> + /*
> + * Validate the invariant that nowait must allow retry
> + * to be sure not to return SIGBUS erroneously on
> + * nowait invocations.
> + */
> + BUG_ON(flags & FAULT_FLAG_RETRY_NOWAIT);
> +#ifdef CONFIG_DEBUG_VM
> + if (printk_ratelimit()) {
> + printk(KERN_WARNING
> + "FAULT_FLAG_ALLOW_RETRY missing %x\n", flags);
> + dump_stack();
> + }
> +#endif
> + return VM_FAULT_SIGBUS;
> + }
> +
> + /*
> + * Handle nowait, not much to do other than tell it to retry
> + * and wait.
> + */
> + if (flags & FAULT_FLAG_RETRY_NOWAIT)
> + return VM_FAULT_RETRY;
> +
> + /* take the reference before dropping the mmap_sem */
> + userfaultfd_ctx_get(ctx);
> +
> + /* be gentle and immediately relinquish the mmap_sem */
> + up_read(&mm->mmap_sem);
> +
> + init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
> + uwq.wq.private = current;
> + uwq.address = userfault_address(address, flags, reason);
> + uwq.pending = true;
> + uwq.ctx = ctx;
> +
> + spin_lock(&ctx->fault_wqh.lock);
> + /*
> + * After the __add_wait_queue the uwq is visible to userland
> + * through poll/read().
> + */
> + __add_wait_queue(&ctx->fault_wqh, &uwq.wq);
> + for (;;) {
> + set_current_state(TASK_KILLABLE);
> + if (!uwq.pending || ACCESS_ONCE(ctx->released) ||
> + fatal_signal_pending(current))
> + break;
> + spin_unlock(&ctx->fault_wqh.lock);
> +
> + wake_up_poll(&ctx->fd_wqh, POLLIN);
> + schedule();
> +
> + spin_lock(&ctx->fault_wqh.lock);
> + }
> + __remove_wait_queue(&ctx->fault_wqh, &uwq.wq);
> + __set_current_state(TASK_RUNNING);
> + spin_unlock(&ctx->fault_wqh.lock);
> +
> + /*
> + * ctx may go away after this if the userfault pseudo fd is
> + * already released.
> + */
> + userfaultfd_ctx_put(ctx);
> +
> + return VM_FAULT_RETRY;
> +}
> +
> +static int userfaultfd_release(struct inode *inode, struct file *file)
> +{
> + struct userfaultfd_ctx *ctx = file->private_data;
> + struct mm_struct *mm = ctx->mm;
> + struct vm_area_struct *vma, *prev;
> + /* len == 0 means wake all */
> + struct userfaultfd_wake_range range = { .len = 0, };
> + unsigned long new_flags;
> +
> + ACCESS_ONCE(ctx->released) = true;
> +
> + /*
> + * Flush page faults out of all CPUs. NOTE: all page faults
> + * must be retried without returning VM_FAULT_SIGBUS if
> + * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
> + * changes while handle_userfault released the mmap_sem. So
> + * it's critical that released is set to true (above), before
> + * taking the mmap_sem for writing.
> + */
> + down_write(&mm->mmap_sem);
> + prev = NULL;
> + for (vma = mm->mmap; vma; vma = vma->vm_next) {
> + cond_resched();
> + BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
> + !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
> + if (vma->vm_userfaultfd_ctx.ctx != ctx) {
> + prev = vma;
> + continue;
> + }
> + new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
> + prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
> + new_flags, vma->anon_vma,
> + vma->vm_file, vma->vm_pgoff,
> + vma_policy(vma),
> + NULL_VM_UFFD_CTX);
> + if (prev)
> + vma = prev;
> + else
> + prev = vma;
> + vma->vm_flags = new_flags;
> + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
> + }
> + up_write(&mm->mmap_sem);
> +
> + /*
> + * After no new page faults can wait on this fault_wqh, flush
> + * the last page faults that may have been already waiting on
> + * the fault_wqh.
> + */
> + spin_lock(&ctx->fault_wqh.lock);
> + __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0, &range);
> + spin_unlock(&ctx->fault_wqh.lock);
> +
> + wake_up_poll(&ctx->fd_wqh, POLLHUP);
> + userfaultfd_ctx_put(ctx);
> + return 0;
> +}
> +
> +static inline unsigned int find_userfault(struct userfaultfd_ctx *ctx,
> + struct userfaultfd_wait_queue **uwq)
> +{
> + wait_queue_t *wq;
> + struct userfaultfd_wait_queue *_uwq;
> + unsigned int ret = 0;
> +
> + spin_lock(&ctx->fault_wqh.lock);
> + list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) {
> + _uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
> + if (_uwq->pending) {
> + ret = POLLIN;
> + if (uwq)
> + *uwq = _uwq;
> + break;
> + }
> + }
> + spin_unlock(&ctx->fault_wqh.lock);
> +
> + return ret;
> +}
> +
> +static unsigned int userfaultfd_poll(struct file *file, poll_table *wait)
> +{
> + struct userfaultfd_ctx *ctx = file->private_data;
> +
> + poll_wait(file, &ctx->fd_wqh, wait);
> +
> + switch (ctx->state) {
> + case UFFD_STATE_WAIT_API:
> + return POLLERR;
> + case UFFD_STATE_RUNNING:
> + return find_userfault(ctx, NULL);
> + default:
> + BUG();
> + }
> +}
> +
> +static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
> + __u64 *addr)
> +{
> + ssize_t ret;
> + DECLARE_WAITQUEUE(wait, current);
> + struct userfaultfd_wait_queue *uwq = NULL;
> +
> + /* always take the fd_wqh lock before the fault_wqh lock */
> + spin_lock(&ctx->fd_wqh.lock);
> + __add_wait_queue(&ctx->fd_wqh, &wait);
> + for (;;) {
> + set_current_state(TASK_INTERRUPTIBLE);
> + if (find_userfault(ctx, &uwq)) {
> + uwq->pending = false;
> + /* careful to always initialize addr if ret == 0 */
> + *addr = uwq->address;
> + ret = 0;
> + break;
> + }
> + if (signal_pending(current)) {
> + ret = -ERESTARTSYS;
> + break;
> + }
> + if (no_wait) {
> + ret = -EAGAIN;
> + break;
> + }
> + spin_unlock(&ctx->fd_wqh.lock);
> + schedule();
> + spin_lock_irq(&ctx->fd_wqh.lock);
> + }
> + __remove_wait_queue(&ctx->fd_wqh, &wait);
> + __set_current_state(TASK_RUNNING);
> + spin_unlock_irq(&ctx->fd_wqh.lock);
> +
> + return ret;
> +}
> +
> +static ssize_t userfaultfd_read(struct file *file, char __user *buf,
> + size_t count, loff_t *ppos)
> +{
> + struct userfaultfd_ctx *ctx = file->private_data;
> + ssize_t _ret, ret = 0;
> + /* careful to always initialize addr if ret == 0 */
> + __u64 uninitialized_var(addr);
> + int no_wait = file->f_flags & O_NONBLOCK;
> +
> + if (ctx->state == UFFD_STATE_WAIT_API)
> + return -EINVAL;
> + BUG_ON(ctx->state != UFFD_STATE_RUNNING);
> +
> + for (;;) {
> + if (count < sizeof(addr))
> + return ret ? ret : -EINVAL;
> + _ret = userfaultfd_ctx_read(ctx, no_wait, &addr);
> + if (_ret < 0)
> + return ret ? ret : _ret;
> + if (put_user(addr, (__u64 __user *) buf))
> + return ret ? ret : -EFAULT;
> + ret += sizeof(addr);
> + buf += sizeof(addr);
> + count -= sizeof(addr);
> + /*
> + * Allow to read more than one fault at time but only
> + * block if waiting for the very first one.
> + */
> + no_wait = O_NONBLOCK;
> + }
> +}
> +
> +static int __wake_userfault(struct userfaultfd_ctx *ctx,
> + struct userfaultfd_wake_range *range)
> +{
> + wait_queue_t *wq;
> + struct userfaultfd_wait_queue *uwq;
> + int ret;
> + unsigned long start, end;
> +
> + start = range->start;
> + end = range->start + range->len;
> +
> + ret = -ENOENT;
> + spin_lock(&ctx->fault_wqh.lock);
> + list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) {
> + uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
> + if (uwq->pending)
> + continue;
> + if (uwq->address >= start && uwq->address < end) {
> + ret = 0;
> + /* wake all in the range and autoremove */
> + __wake_up_locked_key(&ctx->fault_wqh, TASK_NORMAL, 0,
> + range);
> + break;
> + }
> + }
> + spin_unlock(&ctx->fault_wqh.lock);
> +
> + return ret;
> +}
> +
> +static __always_inline int wake_userfault(struct userfaultfd_ctx *ctx,
> + struct userfaultfd_wake_range *range)
> +{
> + if (!waitqueue_active(&ctx->fault_wqh))
> + return -ENOENT;
> +
> + return __wake_userfault(ctx, range);
> +}
> +
> +static __always_inline int validate_range(struct mm_struct *mm,
> + __u64 start, __u64 len)
> +{
> + __u64 task_size = mm->task_size;
> +
> + if (start & ~PAGE_MASK)
> + return -EINVAL;
> + if (len & ~PAGE_MASK)
> + return -EINVAL;
> + if (!len)
> + return -EINVAL;
> + if (start < mmap_min_addr)
> + return -EINVAL;
> + if (start >= task_size)
> + return -EINVAL;
> + if (len > task_size - start)
> + return -EINVAL;
> + return 0;
> +}
> +
> +static int userfaultfd_register(struct userfaultfd_ctx *ctx,
> + unsigned long arg)
> +{
> + struct mm_struct *mm = ctx->mm;
> + struct vm_area_struct *vma, *prev, *cur;
> + int ret;
> + struct uffdio_register uffdio_register;
> + struct uffdio_register __user *user_uffdio_register;
> + unsigned long vm_flags, new_flags;
> + bool found;
> + unsigned long start, end, vma_end;
> +
> + user_uffdio_register = (struct uffdio_register __user *) arg;
> +
> + ret = -EFAULT;
> + if (copy_from_user(&uffdio_register, user_uffdio_register,
> + sizeof(uffdio_register)-sizeof(__u64)))
> + goto out;
> +
> + ret = -EINVAL;
> + if (!uffdio_register.mode)
> + goto out;
> + if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING|
> + UFFDIO_REGISTER_MODE_WP))
> + goto out;
> + vm_flags = 0;
> + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
> + vm_flags |= VM_UFFD_MISSING;
> + if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
> + vm_flags |= VM_UFFD_WP;
> + /*
> + * FIXME: remove the below error constraint by
> + * implementing the wprotect tracking mode.
> + */
> + ret = -EINVAL;
> + goto out;
> + }
> +
> + ret = validate_range(mm, uffdio_register.range.start,
> + uffdio_register.range.len);
> + if (ret)
> + goto out;
> +
> + start = uffdio_register.range.start;
> + end = start + uffdio_register.range.len;
> +
> + down_write(&mm->mmap_sem);
> + vma = find_vma_prev(mm, start, &prev);
> +
> + ret = -ENOMEM;
> + if (!vma)
> + goto out_unlock;
> +
> + /* check that there's at least one vma in the range */
> + ret = -EINVAL;
> + if (vma->vm_start >= end)
> + goto out_unlock;
> +
> + /*
> + * Search for not compatible vmas.
> + *
> + * FIXME: this shall be relaxed later so that it doesn't fail
> + * on tmpfs backed vmas (in addition to the current allowance
> + * on anonymous vmas).
> + */
> + found = false;
> + for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
> + cond_resched();
> +
> + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
> + !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
> +
> + /* check not compatible vmas */
> + ret = -EINVAL;
> + if (cur->vm_ops)
> + goto out_unlock;
> +
> + /*
> + * Check that this vma isn't already owned by a
> + * different userfaultfd. We can't allow more than one
> + * userfaultfd to own a single vma simultaneously or we
> + * wouldn't know which one to deliver the userfaults to.
> + */
> + ret = -EBUSY;
> + if (cur->vm_userfaultfd_ctx.ctx &&
> + cur->vm_userfaultfd_ctx.ctx != ctx)
> + goto out_unlock;
> +
> + found = true;
> + }
> + BUG_ON(!found);
> +
> + /*
> + * Now that we scanned all vmas we can already tell userland which
> + * ioctls methods are guaranteed to succeed on this range.
> + */
> + ret = -EFAULT;
> + if (put_user(UFFD_API_RANGE_IOCTLS, &user_uffdio_register->ioctls))
> + goto out_unlock;
> +
> + if (vma->vm_start < start)
> + prev = vma;
> +
> + ret = 0;
> + do {
> + cond_resched();
> +
> + BUG_ON(vma->vm_ops);
> + BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
> + vma->vm_userfaultfd_ctx.ctx != ctx);
> +
> + /*
> + * Nothing to do: this vma is already registered into this
> + * userfaultfd and with the right tracking mode too.
> + */
> + if (vma->vm_userfaultfd_ctx.ctx == ctx &&
> + (vma->vm_flags & vm_flags) == vm_flags)
> + goto skip;
> +
> + if (vma->vm_start > start)
> + start = vma->vm_start;
> + vma_end = min(end, vma->vm_end);
> +
> + new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
> + prev = vma_merge(mm, prev, start, vma_end, new_flags,
> + vma->anon_vma, vma->vm_file, vma->vm_pgoff,
> + vma_policy(vma),
> + ((struct vm_userfaultfd_ctx){ ctx }));
> + if (prev) {
> + vma = prev;
> + goto next;
> + }
> + if (vma->vm_start < start) {
> + ret = split_vma(mm, vma, start, 1);
> + if (ret)
> + break;
> + }
> + if (vma->vm_end > end) {
> + ret = split_vma(mm, vma, end, 0);
> + if (ret)
> + break;
> + }
> + next:
> + /*
> + * In the vma_merge() successful mprotect-like case 8:
> + * the next vma was merged into the current one and
> + * the current one has not been updated yet.
> + */
> + vma->vm_flags = new_flags;
> + vma->vm_userfaultfd_ctx.ctx = ctx;
> +
> + skip:
> + prev = vma;
> + start = vma->vm_end;
> + vma = vma->vm_next;
> + } while (vma && vma->vm_start < end);
> +out_unlock:
> + up_write(&mm->mmap_sem);
> +out:
> + return ret;
> +}
> +
> +static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
> + unsigned long arg)
> +{
> + struct mm_struct *mm = ctx->mm;
> + struct vm_area_struct *vma, *prev, *cur;
> + int ret;
> + struct uffdio_range uffdio_unregister;
> + unsigned long new_flags;
> + bool found;
> + unsigned long start, end, vma_end;
> + const void __user *buf = (void __user *)arg;
> +
> + ret = -EFAULT;
> + if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
> + goto out;
> +
> + ret = validate_range(mm, uffdio_unregister.start,
> + uffdio_unregister.len);
> + if (ret)
> + goto out;
> +
> + start = uffdio_unregister.start;
> + end = start + uffdio_unregister.len;
> +
> + down_write(&mm->mmap_sem);
> + vma = find_vma_prev(mm, start, &prev);
> +
> + ret = -ENOMEM;
> + if (!vma)
> + goto out_unlock;
> +
> + /* check that there's at least one vma in the range */
> + ret = -EINVAL;
> + if (vma->vm_start >= end)
> + goto out_unlock;
> +
> + /*
> + * Search for not compatible vmas.
> + *
> + * FIXME: this shall be relaxed later so that it doesn't fail
> + * on tmpfs backed vmas (in addition to the current allowance
> + * on anonymous vmas).
> + */
> + found = false;
> + ret = -EINVAL;
> + for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
> + cond_resched();
> +
> + BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
> + !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
> +
> + /*
> + * Check not compatible vmas, not strictly required
> + * here as not compatible vmas cannot have an
> + * userfaultfd_ctx registered on them, but this
> + * provides for more strict behavior to notice
> + * unregistration errors.
> + */
> + if (cur->vm_ops)
> + goto out_unlock;
> +
> + found = true;
> + }
> + BUG_ON(!found);
> +
> + if (vma->vm_start < start)
> + prev = vma;
> +
> + ret = 0;
> + do {
> + cond_resched();
> +
> + BUG_ON(vma->vm_ops);
> +
> + /*
> + * Nothing to do: this vma is already registered into this
> + * userfaultfd and with the right tracking mode too.
> + */
> + if (!vma->vm_userfaultfd_ctx.ctx)
> + goto skip;
> +
> + if (vma->vm_start > start)
> + start = vma->vm_start;
> + vma_end = min(end, vma->vm_end);
> +
> + new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
> + prev = vma_merge(mm, prev, start, vma_end, new_flags,
> + vma->anon_vma, vma->vm_file, vma->vm_pgoff,
> + vma_policy(vma),
> + NULL_VM_UFFD_CTX);
> + if (prev) {
> + vma = prev;
> + goto next;
> + }
> + if (vma->vm_start < start) {
> + ret = split_vma(mm, vma, start, 1);
> + if (ret)
> + break;
> + }
> + if (vma->vm_end > end) {
> + ret = split_vma(mm, vma, end, 0);
> + if (ret)
> + break;
> + }
> + next:
> + /*
> + * In the vma_merge() successful mprotect-like case 8:
> + * the next vma was merged into the current one and
> + * the current one has not been updated yet.
> + */
> + vma->vm_flags = new_flags;
> + vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
> +
> + skip:
> + prev = vma;
> + start = vma->vm_end;
> + vma = vma->vm_next;
> + } while (vma && vma->vm_start < end);
> +out_unlock:
> + up_write(&mm->mmap_sem);
> +out:
> + return ret;
> +}
> +
> +/*
> + * This is mostly needed to re-wakeup those userfaults that were still
> + * pending when userland wake them up the first time. We don't wake
> + * the pending one to avoid blocking reads to block, or non blocking
> + * read to return -EAGAIN, if used with POLLIN, to avoid userland
> + * doubts on why POLLIN wasn't reliable.
> + */
> +static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
> + unsigned long arg)
> +{
> + int ret;
> + struct uffdio_range uffdio_wake;
> + struct userfaultfd_wake_range range;
> + const void __user *buf = (void __user *)arg;
> +
> + ret = -EFAULT;
> + if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
> + goto out;
> +
> + ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
> + if (ret)
> + goto out;
> +
> + range.start = uffdio_wake.start;
> + range.len = uffdio_wake.len;
> +
> + /*
> + * len == 0 means wake all and we don't want to wake all here,
> + * so check it again to be sure.
> + */
> + VM_BUG_ON(!range.len);
> +
> + ret = wake_userfault(ctx, &range);
> +
> +out:
> + return ret;
> +}
> +
> +/*
> + * userland asks for a certain API version and we return which bits
> + * and ioctl commands are implemented in this kernel for such API
> + * version or -EINVAL if unknown.
> + */
> +static int userfaultfd_api(struct userfaultfd_ctx *ctx,
> + unsigned long arg)
> +{
> + struct uffdio_api uffdio_api;
> + void __user *buf = (void __user *)arg;
> + int ret;
> +
> + ret = -EINVAL;
> + if (ctx->state != UFFD_STATE_WAIT_API)
> + goto out;
> + ret = -EFAULT;
> + if (copy_from_user(&uffdio_api, buf, sizeof(__u64)))
> + goto out;
> + if (uffdio_api.api != UFFD_API) {
> + /* careful not to leak info, we only read the first 8 bytes */
> + memset(&uffdio_api, 0, sizeof(uffdio_api));
> + if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
> + goto out;
> + ret = -EINVAL;
> + goto out;
> + }
> + /* careful not to leak info, we only read the first 8 bytes */
> + uffdio_api.bits = UFFD_API_BITS;
> + uffdio_api.ioctls = UFFD_API_IOCTLS;
> + ret = -EFAULT;
> + if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
> + goto out;
> + ctx->state = UFFD_STATE_RUNNING;
> + ret = 0;
> +out:
> + return ret;
> +}
> +
> +static long userfaultfd_ioctl(struct file *file, unsigned cmd,
> + unsigned long arg)
> +{
> + int ret = -EINVAL;
> + struct userfaultfd_ctx *ctx = file->private_data;
> +
> + switch(cmd) {
> + case UFFDIO_API:
> + ret = userfaultfd_api(ctx, arg);
> + break;
> + case UFFDIO_REGISTER:
> + ret = userfaultfd_register(ctx, arg);
> + break;
> + case UFFDIO_UNREGISTER:
> + ret = userfaultfd_unregister(ctx, arg);
> + break;
> + case UFFDIO_WAKE:
> + ret = userfaultfd_wake(ctx, arg);
> + break;
> + }
> + return ret;
> +}
> +
> +#ifdef CONFIG_PROC_FS
> +static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
> +{
> + struct userfaultfd_ctx *ctx = f->private_data;
> + wait_queue_t *wq;
> + struct userfaultfd_wait_queue *uwq;
> + unsigned long pending = 0, total = 0;
> +
> + spin_lock(&ctx->fault_wqh.lock);
> + list_for_each_entry(wq, &ctx->fault_wqh.task_list, task_list) {
> + uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
> + if (uwq->pending)
> + pending++;
> + total++;
> + }
> + spin_unlock(&ctx->fault_wqh.lock);
> +
> + /*
> + * If more protocols will be added, there will be all shown
> + * separated by a space. Like this:
> + * protocols: 0xaa 0xbb
> + */
> + seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
> + pending, total, UFFD_API, UFFD_API_BITS,
> + UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
> +}
> +#endif
> +
> +static const struct file_operations userfaultfd_fops = {
> +#ifdef CONFIG_PROC_FS
> + .show_fdinfo = userfaultfd_show_fdinfo,
> +#endif
> + .release = userfaultfd_release,
> + .poll = userfaultfd_poll,
> + .read = userfaultfd_read,
> + .unlocked_ioctl = userfaultfd_ioctl,
> + .compat_ioctl = userfaultfd_ioctl,
> + .llseek = noop_llseek,
> +};
> +
> +/**
> + * userfaultfd_file_create - Creates an userfaultfd file pointer.
> + * @flags: Flags for the userfaultfd file.
> + *
> + * This function creates an userfaultfd file pointer, w/out installing
> + * it into the fd table. This is useful when the userfaultfd file is
> + * used during the initialization of data structures that require
> + * extra setup after the userfaultfd creation. So the userfaultfd
> + * creation is split into the file pointer creation phase, and the
> + * file descriptor installation phase. In this way races with
> + * userspace closing the newly installed file descriptor can be
> + * avoided. Returns an userfaultfd file pointer, or a proper error
> + * pointer.
> + */
> +static struct file *userfaultfd_file_create(int flags)
> +{
> + struct file *file;
> + struct userfaultfd_ctx *ctx;
> +
> + BUG_ON(!current->mm);
> +
> + /* Check the UFFD_* constants for consistency. */
> + BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
> + BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
> +
> + file = ERR_PTR(-EINVAL);
> + if (flags & ~UFFD_SHARED_FCNTL_FLAGS)
> + goto out;
> +
> + file = ERR_PTR(-ENOMEM);
> + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
> + if (!ctx)
> + goto out;
> +
> + atomic_set(&ctx->refcount, 1);
> + init_waitqueue_head(&ctx->fault_wqh);
> + init_waitqueue_head(&ctx->fd_wqh);
> + ctx->flags = flags;
> + ctx->state = UFFD_STATE_WAIT_API;
> + ctx->released = false;
> + ctx->mm = current->mm;
> + /* prevent the mm struct to be freed */
> + atomic_inc(&ctx->mm->mm_count);
> +
> + file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx,
> + O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
> + if (IS_ERR(file))
> + kfree(ctx);
> +out:
> + return file;
> +}
> +
> +SYSCALL_DEFINE1(userfaultfd, int, flags)
> +{
> + int fd, error;
> + struct file *file;
> +
> + error = get_unused_fd_flags(flags & UFFD_SHARED_FCNTL_FLAGS);
> + if (error < 0)
> + return error;
> + fd = error;
> +
> + file = userfaultfd_file_create(flags);
> + if (IS_ERR(file)) {
> + error = PTR_ERR(file);
> + goto err_put_unused_fd;
> + }
> + fd_install(fd, file);
> +
> + return fd;
> +
> +err_put_unused_fd:
> + put_unused_fd(fd);
> +
> + return error;
> +}
> --
> To unsubscribe from this list: send the line "unsubscribe linux-api" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at http://vger.kernel.org/majordomo-info.html
--
Michael Kerrisk
Linux man-pages maintainer; http://www.kernel.org/doc/man-pages/
Linux/UNIX System Programming Training: http://man7.org/training/
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
