On Fri, Sep 04, 2020 at 02:31:15PM +0300, Adalbert Lazăr wrote:
> From: Mircea Cirjaliu <mcirjaliu@xxxxxxxxxxxxxxx>
>
> Remote mapping creates a mirror VMA that exposes memory owned by another
> process in a zero-copy manner. The pages are mapped in the current process'
> address space with no memory management involved and little impact on the
> remote process operation. Currently incompatible with THP.
>
> Signed-off-by: Mircea Cirjaliu <mcirjaliu@xxxxxxxxxxxxxxx>
> Signed-off-by: Adalbert Lazăr <alazar@xxxxxxxxxxxxxxx>
> ---
> include/linux/remote_mapping.h | 22 +
> include/uapi/linux/remote_mapping.h | 36 +
> mm/Kconfig | 11 +
> mm/Makefile | 1 +
> mm/remote_mapping.c | 1273 +++++++++++++++++++++++++++
> 5 files changed, 1343 insertions(+)
> create mode 100644 include/linux/remote_mapping.h
> create mode 100644 include/uapi/linux/remote_mapping.h
> create mode 100644 mm/remote_mapping.c
>
> diff --git a/include/linux/remote_mapping.h b/include/linux/remote_mapping.h
> new file mode 100644
> index 000000000000..5c1d43e8f669
> --- /dev/null
> +++ b/include/linux/remote_mapping.h
> @@ -0,0 +1,22 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +
> +#ifndef _LINUX_REMOTE_MAPPING_H
> +#define _LINUX_REMOTE_MAPPING_H
> +
> +#include <linux/sched.h>
> +
> +#ifdef CONFIG_REMOTE_MAPPING
> +
> +extern int task_remote_map(struct task_struct *task, int fds[]);
> +
> +#else /* CONFIG_REMOTE_MAPPING */
> +
> +static inline int task_remote_map(struct task_struct *task, int fds[])
> +{
> + return -EINVAL;
> +}
> +
> +#endif /* CONFIG_REMOTE_MAPPING */
> +
> +
> +#endif /* _LINUX_REMOTE_MAPPING_H */
> diff --git a/include/uapi/linux/remote_mapping.h b/include/uapi/linux/remote_mapping.h
> new file mode 100644
> index 000000000000..5d2828a6aa47
> --- /dev/null
> +++ b/include/uapi/linux/remote_mapping.h
> @@ -0,0 +1,36 @@
> +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
> +
> +#ifndef __UAPI_REMOTE_MAPPING_H__
> +#define __UAPI_REMOTE_MAPPING_H__
> +
> +#include <linux/types.h>
> +#include <linux/ioctl.h>
> +
> +// device file interface
> +#define REMOTE_PROC_MAP _IOW('r', 0x01, int)
> +
> +
> +// pidfd interface
> +#define PIDFD_IO_MAGIC 'p'
> +
> +struct pidfd_mem_map {
> + uint64_t address;
> + off_t offset;
> + off_t size;
> + int flags;
> + int padding[7];
> +};
> +
> +struct pidfd_mem_unmap {
> + off_t offset;
> + off_t size;
> +};
> +
> +#define PIDFD_MEM_MAP _IOW(PIDFD_IO_MAGIC, 0x01, struct pidfd_mem_map)
> +#define PIDFD_MEM_UNMAP _IOW(PIDFD_IO_MAGIC, 0x02, struct pidfd_mem_unmap)
> +#define PIDFD_MEM_LOCK _IOW(PIDFD_IO_MAGIC, 0x03, int)
> +
> +#define PIDFD_MEM_REMAP _IOW(PIDFD_IO_MAGIC, 0x04, unsigned long)
> +// TODO: actually this is not for pidfd, find better names
> +
> +#endif /* __UAPI_REMOTE_MAPPING_H__ */
> diff --git a/mm/Kconfig b/mm/Kconfig
> index c1acc34c1c35..0ecc3f41a98e 100644
> --- a/mm/Kconfig
> +++ b/mm/Kconfig
> @@ -804,6 +804,17 @@ config HMM_MIRROR
> bool
> depends on MMU
>
> +config REMOTE_MAPPING
> + bool "Remote memory mapping"
> + depends on X86_64 && MMU && !TRANSPARENT_HUGEPAGE
> + select MMU_NOTIFIER
> + default n
> + help
> + Allows a client process to gain access to an unrelated process'
> + address space on a range-basis. The feature maps pages found at
> + the remote equivalent address in the current process' page tables
> + in a lightweight manner.
> +
> config DEVICE_PRIVATE
> bool "Unaddressable device memory (GPU memory, ...)"
> depends on ZONE_DEVICE
> diff --git a/mm/Makefile b/mm/Makefile
> index fccd3756b25f..ce1a00e7bc8c 100644
> --- a/mm/Makefile
> +++ b/mm/Makefile
> @@ -112,3 +112,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
> obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
> obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
> obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
> +obj-$(CONFIG_REMOTE_MAPPING) += remote_mapping.o
> diff --git a/mm/remote_mapping.c b/mm/remote_mapping.c
> new file mode 100644
> index 000000000000..1dc53992424b
> --- /dev/null
> +++ b/mm/remote_mapping.c
> @@ -0,0 +1,1273 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Remote memory mapping.
> + *
> + * Copyright (C) 2017-2020 Bitdefender S.R.L.
> + *
> + * Author:
> + * Mircea Cirjaliu <mcirjaliu@xxxxxxxxxxxxxxx>
> + */
> +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
> +
> +#include <linux/init.h>
> +#include <linux/kernel.h>
> +#include <linux/spinlock.h>
> +#include <linux/mutex.h>
> +#include <linux/module.h>
> +#include <linux/printk.h>
> +#include <linux/mm.h>
> +#include <linux/mman.h>
> +#include <linux/pid.h>
> +#include <linux/file.h>
> +#include <linux/mmu_notifier.h>
> +#include <linux/sched/task.h>
> +#include <linux/sched/mm.h>
> +#include <linux/sched/signal.h>
> +#include <linux/interval_tree_generic.h>
> +#include <linux/refcount.h>
> +#include <linux/miscdevice.h>
> +#include <uapi/linux/remote_mapping.h>
> +#include <linux/pfn_t.h>
> +#include <linux/errno.h>
> +#include <linux/limits.h>
> +#include <linux/anon_inodes.h>
> +#include <linux/fdtable.h>
> +#include <asm/tlb.h>
> +#include "internal.h"
> +
> +struct remote_file_context {
> + refcount_t refcount;
> +
> + struct srcu_struct fault_srcu;
> + struct mm_struct *mm;
> +
> + bool locked;
> + struct rb_root_cached rb_view; /* view offset tree */
> + struct mutex views_lock;
> +
> +};
> +
> +struct remote_view {
> + refcount_t refcount;
> +
> + unsigned long address;
> + unsigned long size;
> + unsigned long offset;
> + bool valid;
> +
> + struct rb_node target_rb; /* link for views tree */
> + unsigned long rb_subtree_last; /* in remote_file_context */
> +
> + struct mmu_interval_notifier mmin;
> + spinlock_t user_lock;
> +
> + /*
> + * interval tree for mapped ranges (indexed by source process HVA)
> + * because of GPA->HVA aliasing, multiple ranges may overlap
> + */
> + struct rb_root_cached rb_rmap; /* rmap tree */
> + struct rw_semaphore rmap_lock;
> +};
> +
> +struct remote_vma_context {
> + struct vm_area_struct *vma; /* link back to VMA */
> + struct remote_view *view; /* corresponding view */
> +
> + struct rb_node rmap_rb; /* link for rmap tree */
> + unsigned long rb_subtree_last;
> +};
> +
> +/* view offset tree */
> +static inline unsigned long view_start(struct remote_view *view)
> +{
> + return view->offset + 1;
> +}
> +
> +static inline unsigned long view_last(struct remote_view *view)
> +{
> + return view->offset + view->size - 1;
> +}
> +
> +INTERVAL_TREE_DEFINE(struct remote_view, target_rb,
> + unsigned long, rb_subtree_last, view_start, view_last,
> + static inline, view_interval_tree)
> +
> +#define view_tree_foreach(view, root, start, last) \
> + for (view = view_interval_tree_iter_first(root, start, last); \
> + view; view = view_interval_tree_iter_next(view, start, last))
> +
> +/* rmap interval tree */
> +static inline unsigned long ctx_start(struct remote_vma_context *ctx)
> +{
> + struct vm_area_struct *vma = ctx->vma;
> + struct remote_view *view = ctx->view;
> + unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
> +
> + return offset - view->offset + view->address;
> +}
> +
> +static inline unsigned long ctx_last(struct remote_vma_context *ctx)
> +{
> + struct vm_area_struct *vma = ctx->vma;
> + struct remote_view *view = ctx->view;
> + unsigned long offset;
> +
> + offset = (vma->vm_pgoff << PAGE_SHIFT) + (vma->vm_end - vma->vm_start);
> +
> + return offset - view->offset + view->address;
> +}
> +
> +static inline unsigned long ctx_rmap_start(struct remote_vma_context *ctx)
> +{
> + return ctx_start(ctx) + 1;
> +}
> +
> +static inline unsigned long ctx_rmap_last(struct remote_vma_context *ctx)
> +{
> + return ctx_last(ctx) - 1;
> +}
> +
> +INTERVAL_TREE_DEFINE(struct remote_vma_context, rmap_rb,
> + unsigned long, rb_subtree_last, ctx_rmap_start, ctx_rmap_last,
> + static inline, rmap_interval_tree)
> +
> +#define rmap_foreach(ctx, root, start, last) \
> + for (ctx = rmap_interval_tree_iter_first(root, start, last); \
> + ctx; ctx = rmap_interval_tree_iter_next(ctx, start, last))
> +
> +static int mirror_zap_pte(struct vm_area_struct *vma, unsigned long addr,
> + pte_t *pte, int rss[], struct mmu_gather *tlb,
> + struct zap_details *details)
> +{
> + pte_t ptent = *pte;
> + struct page *page;
> + int flags = 0;
> +
> + page = vm_normal_page(vma, addr, ptent);
> + //ptent = ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);
> + ptent = ptep_clear_flush_notify(vma, addr, pte);
> + //tlb_remove_tlb_entry(tlb, pte, addr);
> +
> + if (pte_dirty(ptent)) {
> + flags |= ZAP_PTE_FLUSH;
> + set_page_dirty(page);
> + }
> +
> + return flags;
> +}
> +
> +static void
> +zap_remote_range(struct vm_area_struct *vma,
> + unsigned long start, unsigned long end,
> + bool atomic)
> +{
> + struct mmu_notifier_range range;
> + struct mmu_gather tlb;
> + struct zap_details details = {
> + .atomic = atomic,
> + };
> +
> + pr_debug("%s: vma %lx-%lx, zap range %lx-%lx\n",
> + __func__, vma->vm_start, vma->vm_end, start, end);
> +
> + tlb_gather_mmu(&tlb, vma->vm_mm, start, end);
> +
> + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0,
> + vma, vma->vm_mm, start, end);
> + if (atomic)
> + mmu_notifier_invalidate_range_start_nonblock(&range);
> + else
> + mmu_notifier_invalidate_range_start(&range);
> +
> + unmap_page_range(&tlb, vma, start, end, &details);
> +
> + mmu_notifier_invalidate_range_end(&range);
> + tlb_finish_mmu(&tlb, start, end);
> +}
> +
> +static bool
> +mirror_clear_view(struct remote_view *view,
> + unsigned long start, unsigned long last, bool atomic)
> +{
> + struct remote_vma_context *ctx;
> + unsigned long src_start, src_last;
> + unsigned long vma_start, vma_last;
> +
> + pr_debug("%s: view %p [%lx-%lx), range [%lx-%lx)", __func__, view,
> + view->offset, view->offset + view->size, start, last);
> +
> + if (likely(!atomic))
> + down_read(&view->rmap_lock);
> + else if (!down_read_trylock(&view->rmap_lock))
> + return false;
> +
> + rmap_foreach(ctx, &view->rb_rmap, start, last) {
> + struct vm_area_struct *vma = ctx->vma;
> +
> + // intersect intervals (source process address range)
> + src_start = max(start, ctx_start(ctx));
> + src_last = min(last, ctx_last(ctx));
> +
> + // translate to destination process address range
> + vma_start = vma->vm_start + (src_start - ctx_start(ctx));
> + vma_last = vma->vm_end - (ctx_last(ctx) - src_last);
> +
> + zap_remote_range(vma, vma_start, vma_last, atomic);
> + }
> +
> + up_read(&view->rmap_lock);
> +
> + return true;
> +}
> +
> +static bool mmin_invalidate(struct mmu_interval_notifier *interval_sub,
> + const struct mmu_notifier_range *range,
> + unsigned long cur_seq)
> +{
> + struct remote_view *view =
> + container_of(interval_sub, struct remote_view, mmin);
> +
> + pr_debug("%s: reason %d, range [%lx-%lx)\n", __func__,
> + range->event, range->start, range->end);
> +
> + spin_lock(&view->user_lock);
> + mmu_interval_set_seq(interval_sub, cur_seq);
> + spin_unlock(&view->user_lock);
> +
> + /* mark view as invalid before zapping the page tables */
> + if (range->event == MMU_NOTIFY_RELEASE)
> + WRITE_ONCE(view->valid, false);
> +
> + return mirror_clear_view(view, range->start, range->end,
> + !mmu_notifier_range_blockable(range));
> +}
> +
> +static const struct mmu_interval_notifier_ops mmin_ops = {
> + .invalidate = mmin_invalidate,
> +};
> +
> +static void view_init(struct remote_view *view)
> +{
> + refcount_set(&view->refcount, 1);
> + view->valid = true;
> + RB_CLEAR_NODE(&view->target_rb);
> + view->rb_rmap = RB_ROOT_CACHED;
> + init_rwsem(&view->rmap_lock);
> + spin_lock_init(&view->user_lock);
> +}
> +
> +/* return working view or reason why it failed */
> +static struct remote_view *
> +view_alloc(struct mm_struct *mm, unsigned long address, unsigned long size, unsigned long offset)
> +{
> + struct remote_view *view;
> + int result;
> +
> + view = kzalloc(sizeof(*view), GFP_KERNEL);
> + if (!view)
> + return ERR_PTR(-ENOMEM);
> +
> + view_init(view);
> +
> + view->address = address;
> + view->size = size;
> + view->offset = offset;
> +
> + pr_debug("%s: view %p [%lx-%lx)", __func__, view,
> + view->offset, view->offset + view->size);
> +
> + result = mmu_interval_notifier_insert(&view->mmin, mm, address, size, &mmin_ops);
> + if (result) {
> + kfree(view);
> + return ERR_PTR(result);
> + }
> +
> + return view;
> +}
> +
> +static void
> +view_insert(struct remote_file_context *fctx, struct remote_view *view)
> +{
> + view_interval_tree_insert(view, &fctx->rb_view);
> + refcount_inc(&view->refcount);
> +}
> +
> +static struct remote_view *
> +view_search_get(struct remote_file_context *fctx,
> + unsigned long start, unsigned long last)
> +{
> + struct remote_view *view;
> +
> + lockdep_assert_held(&fctx->views_lock);
> +
> + /*
> + * loop & return the first view intersecting interval
> + * further checks will be done down the road
> + */
> + view_tree_foreach(view, &fctx->rb_view, start, last)
> + break;
> +
> + if (view)
> + refcount_inc(&view->refcount);
> +
> + return view;
> +}
> +
> +static void
> +view_put(struct remote_view *view)
> +{
> + if (refcount_dec_and_test(&view->refcount)) {
> + pr_debug("%s: view %p [%lx-%lx) bye bye", __func__, view,
> + view->offset, view->offset + view->size);
> +
> + mmu_interval_notifier_remove(&view->mmin);
> + kfree(view);
> + }
> +}
> +
> +static void
> +view_remove(struct remote_file_context *fctx, struct remote_view *view)
> +{
> + view_interval_tree_remove(view, &fctx->rb_view);
> + RB_CLEAR_NODE(&view->target_rb);
> + view_put(view);
> +}
> +
> +static bool
> +view_overlaps(struct remote_file_context *fctx,
> + unsigned long start, unsigned long last)
> +{
> + struct remote_view *view;
> +
> + view_tree_foreach(view, &fctx->rb_view, start, last)
> + return true;
> +
> + return false;
> +}
> +
> +static struct remote_view *
> +alloc_identity_view(struct mm_struct *mm)
> +{
> + return view_alloc(mm, 0, ULONG_MAX, 0);
> +}
> +
> +static void remote_file_context_init(struct remote_file_context *fctx)
> +{
> + refcount_set(&fctx->refcount, 1);
> + init_srcu_struct(&fctx->fault_srcu);
> + fctx->locked = false;
> + fctx->rb_view = RB_ROOT_CACHED;
> + mutex_init(&fctx->views_lock);
> +}
> +
> +static struct remote_file_context *remote_file_context_alloc(void)
> +{
> + struct remote_file_context *fctx;
> +
> + fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
> + if (fctx)
> + remote_file_context_init(fctx);
> +
> + pr_debug("%s: fctx %p\n", __func__, fctx);
> +
> + return fctx;
> +}
> +
> +static void remote_file_context_get(struct remote_file_context *fctx)
> +{
> + refcount_inc(&fctx->refcount);
> +}
> +
> +static void remote_file_context_put(struct remote_file_context *fctx)
> +{
> + struct remote_view *view, *n;
> +
> + if (refcount_dec_and_test(&fctx->refcount)) {
> + pr_debug("%s: fctx %p\n", __func__, fctx);
> +
> + rbtree_postorder_for_each_entry_safe(view, n,
> + &fctx->rb_view.rb_root, target_rb)
> + view_put(view);
> +
> + if (fctx->mm)
> + mmdrop(fctx->mm);
> +
> + kfree(fctx);
> + }
> +}
> +
> +static void remote_vma_context_init(struct remote_vma_context *ctx)
> +{
> + RB_CLEAR_NODE(&ctx->rmap_rb);
> +}
> +
> +static struct remote_vma_context *remote_vma_context_alloc(void)
> +{
> + struct remote_vma_context *ctx;
> +
> + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
> + if (ctx)
> + remote_vma_context_init(ctx);
> +
> + return ctx;
> +}
> +
> +static void remote_vma_context_free(struct remote_vma_context *ctx)
> +{
> + kfree(ctx);
> +}
> +
> +static int mirror_dev_open(struct inode *inode, struct file *file)
> +{
> + struct remote_file_context *fctx;
> +
> + pr_debug("%s: file %p\n", __func__, file);
> +
> + fctx = remote_file_context_alloc();
> + if (!fctx)
> + return -ENOMEM;
> + file->private_data = fctx;
> +
> + return 0;
> +}
> +
> +static int do_remote_proc_map(struct file *file, int pid)
> +{
> + struct remote_file_context *fctx = file->private_data;
> + struct task_struct *req_task;
> + struct mm_struct *req_mm;
> + struct remote_view *id;
> + int result = 0;
> +
> + pr_debug("%s: pid %d\n", __func__, pid);
> +
> + req_task = find_get_task_by_vpid(pid);
> + if (!req_task)
> + return -ESRCH;
> +
> + req_mm = get_task_mm(req_task);
> + put_task_struct(req_task);
> + if (!req_mm)
> + return -EINVAL;
> +
> + /* error on 2nd call or multithreaded race */
> + if (cmpxchg(&fctx->mm, (struct mm_struct *)NULL, req_mm) != NULL) {
> + result = -EALREADY;
> + goto out;
> + } else
> + mmgrab(req_mm);
> +
> + id = alloc_identity_view(req_mm);
> + if (IS_ERR(id)) {
> + mmdrop(req_mm);
> + result = PTR_ERR(id);
> + goto out;
> + }
> +
> + /* one view only, don't need to take mutex */
> + view_insert(fctx, id);
> + view_put(id); /* usage reference */
> +
> +out:
> + mmput(req_mm);
> +
> + return result;
> +}
> +
> +static long mirror_dev_ioctl(struct file *file, unsigned int ioctl,
> + unsigned long arg)
> +{
> + long result;
> +
> + switch (ioctl) {
> + case REMOTE_PROC_MAP: {
> + int pid = (int)arg;
> +
> + result = do_remote_proc_map(file, pid);
> + break;
> + }
> +
> + default:
> + pr_debug("%s: ioctl %x not implemented\n", __func__, ioctl);
> + result = -ENOTTY;
> + }
> +
> + return result;
> +}
> +
> +/*
> + * This is called after all reference to the file have been dropped,
> + * including mmap()s, even if the file is close()d first.
> + */
> +static int mirror_dev_release(struct inode *inode, struct file *file)
> +{
> + struct remote_file_context *fctx = file->private_data;
> +
> + pr_debug("%s: file %p\n", __func__, file);
> +
> + remote_file_context_put(fctx);
> +
> + return 0;
> +}
> +
> +static struct page *mm_remote_get_page(struct mm_struct *req_mm,
> + unsigned long address, unsigned int flags)
> +{
> + struct page *req_page = NULL;
> + long nrpages;
> +
> + might_sleep();
> +
> + flags |= FOLL_ANON | FOLL_MIGRATION;
> +
> + /* get host page corresponding to requested address */
> + nrpages = get_user_pages_remote(NULL, req_mm, address, 1,
> + flags, &req_page, NULL, NULL);
> + if (unlikely(nrpages == 0)) {
> + pr_err("no page at %lx\n", address);
> + return ERR_PTR(-ENOENT);
> + }
> + if (IS_ERR_VALUE(nrpages)) {
> + pr_err("get_user_pages_remote() failed: %d\n", (int)nrpages);
> + return ERR_PTR(nrpages);
> + }
> +
> + /* limit introspection to anon memory (this also excludes zero-page) */
> + if (!PageAnon(req_page)) {
> + put_page(req_page);
> + pr_err("page at %lx not anon\n", address);
> + return ERR_PTR(-EINVAL);
> + }
> +
> + return req_page;
> +}
> +
> +/*
> + * avoid PTE allocation in this function for 2 reasons:
> + * - it runs under user_lock, which is a spinlock and can't sleep
> + * (user_lock can be a mutex is allocation is needed)
> + * - PTE allocation triggers reclaim, which causes a possible deadlock warning
> + */
> +static vm_fault_t remote_map_page(struct vm_fault *vmf, struct page *page)
> +{
> + struct vm_area_struct *vma = vmf->vma;
> + pte_t entry;
> +
> + if (vmf->prealloc_pte) {
> + vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
> + if (unlikely(!pmd_none(*vmf->pmd))) {
> + spin_unlock(vmf->ptl);
> + goto map_pte;
> + }
> +
> + mm_inc_nr_ptes(vma->vm_mm);
> + pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
> + spin_unlock(vmf->ptl);
> + vmf->prealloc_pte = NULL;
> + } else {
> + BUG_ON(pmd_none(*vmf->pmd));
> + }
> +
> +map_pte:
> + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl);
> +
> + if (!pte_none(*vmf->pte))
> + goto out_unlock;
> +
> + entry = mk_pte(page, vma->vm_page_prot);
> + set_pte_at_notify(vma->vm_mm, vmf->address, vmf->pte, entry);
> +
> +out_unlock:
> + pte_unmap_unlock(vmf->pte, vmf->ptl);
> + return VM_FAULT_NOPAGE;
> +}
> +
> +static vm_fault_t mirror_vm_fault(struct vm_fault *vmf)
> +{
> + struct vm_area_struct *vma = vmf->vma;
> + struct mm_struct *mm = vma->vm_mm;
> + struct remote_vma_context *ctx = vma->vm_private_data;
> + struct remote_view *view = ctx->view;
> + struct file *file = vma->vm_file;
> + struct remote_file_context *fctx = file->private_data;
> + unsigned long req_addr;
> + unsigned int gup_flags;
> + struct page *req_page;
> + vm_fault_t result = VM_FAULT_SIGBUS;
> + struct mm_struct *src_mm = fctx->mm;
> + unsigned long seq;
> + int idx;
> +
> +fault_retry:
> + seq = mmu_interval_read_begin(&view->mmin);
> +
> + idx = srcu_read_lock(&fctx->fault_srcu);
> +
> + /* check if view was invalidated */
> + if (unlikely(!READ_ONCE(view->valid))) {
> + pr_debug("%s: region [%lx-%lx) was invalidated!!\n", __func__,
> + view->offset, view->offset + view->size);
> + goto out_invalid; /* VM_FAULT_SIGBUS */
> + }
> +
> + /* drop current mm semapchore */
> + up_read(¤t->mm->mmap_sem);
> +
> + /* take remote mm semaphore */
> + if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) {
> + if (!down_read_trylock(&src_mm->mmap_sem)) {
> + pr_debug("%s: couldn't take source semaphore!!\n", __func__);
> + goto out_retry;
> + }
> + } else
> + down_read(&src_mm->mmap_sem);
> +
> + /* set GUP flags depending on the VMA */
> + gup_flags = 0;
> + if (vma->vm_flags & VM_WRITE)
> + gup_flags |= FOLL_WRITE | FOLL_FORCE;
> +
> + /* translate file offset to source process HVA */
> + req_addr = (vmf->pgoff << PAGE_SHIFT) - view->offset + view->address;
> + req_page = mm_remote_get_page(src_mm, req_addr, gup_flags);
> +
> + /* check for validity of the page */
> + if (IS_ERR_OR_NULL(req_page)) {
> + up_read(&src_mm->mmap_sem);
> +
> + if (PTR_ERR(req_page) == -ERESTARTSYS ||
> + PTR_ERR(req_page) == -EBUSY) {
> + goto out_retry;
> + } else
> + goto out_err; /* VM_FAULT_SIGBUS */
> + }
> +
> + up_read(&src_mm->mmap_sem);
> +
> + /* retake current mm semapchore */
> + down_read(¤t->mm->mmap_sem);
> +
> + /* expedite retry */
> + if (mmu_interval_check_retry(&view->mmin, seq)) {
> + put_page(req_page);
> +
> + srcu_read_unlock(&fctx->fault_srcu, idx);
> +
> + goto fault_retry;
> + }
> +
> + /* make sure the VMA hasn't gone away */
> + vma = find_vma(current->mm, vmf->address);
> + if (vma == vmf->vma) {
> + spin_lock(&view->user_lock);
> +
> + if (mmu_interval_read_retry(&view->mmin, seq)) {
> + spin_unlock(&view->user_lock);
> +
> + put_page(req_page);
> +
> + srcu_read_unlock(&fctx->fault_srcu, idx);
> +
> + goto fault_retry;
> + }
> +
> + result = remote_map_page(vmf, req_page); /* install PTE here */
> +
> + spin_unlock(&view->user_lock);
> + }
> +
> + put_page(req_page);
> +
> + srcu_read_unlock(&fctx->fault_srcu, idx);
> +
> + return result;
> +
> +out_err:
> + /* retake current mm semapchore */
> + down_read(¤t->mm->mmap_sem);
> +out_invalid:
> + srcu_read_unlock(&fctx->fault_srcu, idx);
> +
> + return result;
> +
> +out_retry:
> + /* retake current mm semapchore */
> + down_read(¤t->mm->mmap_sem);
> +
> + srcu_read_unlock(&fctx->fault_srcu, idx);
> +
> + /* TODO: some optimizations work here when we arrive with FAULT_FLAG_ALLOW_RETRY */
> + /* TODO: mmap_sem doesn't need to be taken, then dropped */
> +
> + /*
> + * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released
> + * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
> + * not set.
> + *
> + * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
> + * set, VM_FAULT_RETRY can still be returned if and only if there are
> + * fatal_signal_pending()s, and the mmap_sem must be released before
> + * returning it.
> + */
> + if (vmf->flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_TRIED)) {
> + if (!(vmf->flags & FAULT_FLAG_KILLABLE))
> + if (current && fatal_signal_pending(current)) {
> + up_read(¤t->mm->mmap_sem);
> + return VM_FAULT_RETRY;
> + }
> +
> + if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
> + up_read(&mm->mmap_sem);
> +
> + return VM_FAULT_RETRY;
> + } else
> + return VM_FAULT_SIGBUS;
> +}
> +
> +/*
> + * This is called in remove_vma() at the end of __do_munmap() after the address
> + * space has been unmapped and the page tables have been freed.
> + */
> +static void mirror_vm_close(struct vm_area_struct *vma)
> +{
> + struct remote_vma_context *ctx = vma->vm_private_data;
> + struct remote_view *view = ctx->view;
> +
> + pr_debug("%s: VMA %lx-%lx (%lu bytes)\n", __func__,
> + vma->vm_start, vma->vm_end, vma->vm_end - vma->vm_start);
> +
> + /* will wait for any running invalidate notifiers to finish */
> + down_write(&view->rmap_lock);
> + rmap_interval_tree_remove(ctx, &view->rb_rmap);
> + up_write(&view->rmap_lock);
> + view_put(view);
> +
> + remote_vma_context_free(ctx);
> +}
> +
> +/* prevent partial unmap of destination VMA */
> +static int mirror_vm_split(struct vm_area_struct *area, unsigned long addr)
> +{
> + return -EINVAL;
> +}
> +
> +static const struct vm_operations_struct mirror_vm_ops = {
> + .close = mirror_vm_close,
> + .fault = mirror_vm_fault,
> + .split = mirror_vm_split,
> + .zap_pte = mirror_zap_pte,
> +};
> +
> +static bool is_mirror_vma(struct vm_area_struct *vma)
> +{
> + return vma->vm_ops == &mirror_vm_ops;
> +}
> +
> +static struct remote_view *
> +getme_matching_view(struct remote_file_context *fctx,
> + unsigned long start, unsigned long last)
> +{
> + struct remote_view *view;
> +
> + /* lookup view for the VMA offset range */
> + view = view_search_get(fctx, start, last);
> + if (!view)
> + return NULL;
> +
> + /* make sure the interval we're after is contained in the view */
> + if (start < view->offset || last > view->offset + view->size) {
> + view_put(view);
> + return NULL;
> + }
> +
> + return view;
> +}
> +
> +static struct remote_view *
> +getme_exact_view(struct remote_file_context *fctx,
> + unsigned long start, unsigned long last)
> +{
> + struct remote_view *view;
> +
> + /* lookup view for the VMA offset range */
> + view = view_search_get(fctx, start, last);
> + if (!view)
> + return NULL;
> +
> + /* make sure the interval we're after is contained in the view */
> + if (start != view->offset || last != view->offset + view->size) {
> + view_put(view);
> + return NULL;
> + }
> +
> + return view;
> +}
> +
> +static int mirror_dev_mmap(struct file *file, struct vm_area_struct *vma)
> +{
> + struct remote_file_context *fctx = file->private_data;
> + struct remote_vma_context *ctx;
> + unsigned long start, length, last;
> + struct remote_view *view;
> +
> + start = vma->vm_pgoff << PAGE_SHIFT;
> + length = vma->vm_end - vma->vm_start;
> + last = start + length;
> +
> + pr_debug("%s: VMA %lx-%lx (%lu bytes), offsets %lx-%lx\n", __func__,
> + vma->vm_start, vma->vm_end, length, start, last);
> +
> + if (!(vma->vm_flags & VM_SHARED)) {
> + pr_debug("%s: VMA is not shared\n", __func__);
> + return -EINVAL;
> + }
> +
> + /* prepare the context */
> + ctx = remote_vma_context_alloc();
> + if (!ctx)
> + return -ENOMEM;
> +
> + /* lookup view for the VMA offset range */
> + mutex_lock(&fctx->views_lock);
> + view = getme_matching_view(fctx, start, last);
> + mutex_unlock(&fctx->views_lock);
> + if (!view) {
> + pr_debug("%s: no view for range %lx-%lx\n", __func__, start, last);
> + remote_vma_context_free(ctx);
> + return -EINVAL;
> + }
> +
> + /* VMA must be linked to ctx before adding to rmap tree !! */
> + vma->vm_private_data = ctx;
> + ctx->vma = vma;
> +
> + /* view may already be invalidated by the time it's linked */
> + down_write(&view->rmap_lock);
> + ctx->view = view; /* view reference goes here */
> + rmap_interval_tree_insert(ctx, &view->rb_rmap);
> + up_write(&view->rmap_lock);
> +
> + /* set basic VMA properties */
> + vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND;
> + vma->vm_ops = &mirror_vm_ops;
> +
> + return 0;
> +}
> +
> +static const struct file_operations mirror_ops = {
> + .open = mirror_dev_open,
> + .unlocked_ioctl = mirror_dev_ioctl,
> + .compat_ioctl = mirror_dev_ioctl,
> + .llseek = no_llseek,
> + .mmap = mirror_dev_mmap,
> + .release = mirror_dev_release,
> +};
> +
> +static struct miscdevice mirror_dev = {
> + .minor = MISC_DYNAMIC_MINOR,
> + .name = "mirror-proc",
> + .fops = &mirror_ops,
> +};
> +
> +builtin_misc_device(mirror_dev);
> +
> +static int pidfd_mem_remap(struct remote_file_context *fctx, unsigned long address)
> +{
> + struct vm_area_struct *vma;
> + unsigned long start, last;
> + struct remote_vma_context *ctx;
> + struct remote_view *view, *new_view;
> + int result = 0;
> +
> + pr_debug("%s: address %lx\n", __func__, address);
> +
> + down_write(¤t->mm->mmap_sem);
> +
> + vma = find_vma(current->mm, address);
> + if (!vma || !is_mirror_vma(vma)) {
> + result = -EINVAL;
> + goto out_vma;
> + }
> +
> + ctx = vma->vm_private_data;
> + view = ctx->view;
> +
> + if (view->valid)
> + goto out_vma;
> +
> + start = vma->vm_pgoff << PAGE_SHIFT;
> + last = start + (vma->vm_end - vma->vm_start);
> +
> + /* lookup view for the VMA offset range */
> + mutex_lock(&fctx->views_lock);
> + new_view = getme_matching_view(fctx, start, last);
> + mutex_unlock(&fctx->views_lock);
> + if (!new_view) {
> + result = -EINVAL;
> + goto out_vma;
> + }
> + /* do not link to another invalid view */
> + if (!new_view->valid) {
> + view_put(new_view);
> + result = -EINVAL;
> + goto out_vma;
> + }
> +
> + /* we have current->mm->mmap_sem in write mode, so no faults going on */
> + down_write(&view->rmap_lock);
> + rmap_interval_tree_remove(ctx, &view->rb_rmap);
> + up_write(&view->rmap_lock);
> + view_put(view); /* ctx reference */
> +
> + /* replace with the new view */
> + down_write(&new_view->rmap_lock);
> + ctx->view = new_view; /* new view reference goes here */
> + rmap_interval_tree_insert(ctx, &new_view->rb_rmap);
> + up_write(&new_view->rmap_lock);
> +
> +out_vma:
> + up_write(¤t->mm->mmap_sem);
> +
> + return result;
> +}
> +
> +static long
> +pidfd_mem_map_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
> +{
> + struct remote_file_context *fctx = file->private_data;
> + long result = 0;
> +
> + switch (ioctl) {
> + case PIDFD_MEM_REMAP:
> + result = pidfd_mem_remap(fctx, arg);
> + break;
> +
> + default:
> + pr_debug("%s: ioctl %x not implemented\n", __func__, ioctl);
> + result = -ENOTTY;
> + }
> +
> + return result;
> +}
> +
> +static void pidfd_mem_lock(struct remote_file_context *fctx)
> +{
> + pr_debug("%s:\n", __func__);
> +
> + mutex_lock(&fctx->views_lock);
> + fctx->locked = true;
> + mutex_unlock(&fctx->views_lock);
> +}
> +
> +static int pidfd_mem_map(struct remote_file_context *fctx, struct pidfd_mem_map *map)
> +{
> + struct remote_view *view;
> + int result = 0;
> +
> + pr_debug("%s: offset %lx, size %lx, address %lx\n",
> + __func__, map->offset, map->size, (long)map->address);
> +
> + if (!PAGE_ALIGNED(map->offset))
> + return -EINVAL;
> + if (!PAGE_ALIGNED(map->size))
> + return -EINVAL;
> + if (!PAGE_ALIGNED(map->address))
> + return -EINVAL;
> +
> + /* make sure we're creating the view for a valid address space */
> + if (!mmget_not_zero(fctx->mm))
> + return -EINVAL;
> +
> + view = view_alloc(fctx->mm, map->address, map->size, map->offset);
> + if (IS_ERR(view)) {
> + result = PTR_ERR(view);
> + goto out_mm;
> + }
> +
> + mutex_lock(&fctx->views_lock);
> +
> + /* locked ? */
> + if (unlikely(fctx->locked)) {
> + pr_debug("%s: views locked\n", __func__);
> + result = -EINVAL;
> + goto out;
> + }
> +
> + /* overlaps another view ? */
> + if (view_overlaps(fctx, map->offset, map->offset + map->size)) {
> + pr_debug("%s: range overlaps\n", __func__);
> + result = -EALREADY;
> + goto out;
> + }
> +
> + view_insert(fctx, view);
> +
> +out:
> + mutex_unlock(&fctx->views_lock);
> +
> + view_put(view); /* usage reference */
> +out_mm:
> + mmput(fctx->mm);
> +
> + return result;
> +}
> +
> +static int pidfd_mem_unmap(struct remote_file_context *fctx, struct pidfd_mem_unmap *unmap)
> +{
> + struct remote_view *view;
> +
> + pr_debug("%s: offset %lx, size %lx\n",
> + __func__, unmap->offset, unmap->size);
> +
> + if (!PAGE_ALIGNED(unmap->offset))
> + return -EINVAL;
> + if (!PAGE_ALIGNED(unmap->size))
> + return -EINVAL;
> +
> + mutex_lock(&fctx->views_lock);
> +
> + if (unlikely(fctx->locked)) {
> + mutex_unlock(&fctx->views_lock);
> + return -EINVAL;
> + }
> +
> + view = getme_exact_view(fctx, unmap->offset, unmap->offset + unmap->size);
> + if (!view) {
> + mutex_unlock(&fctx->views_lock);
> + return -EINVAL;
> + }
> +
> + view_remove(fctx, view);
> +
> + mutex_unlock(&fctx->views_lock);
> +
> + /*
> + * The view may still be refernced by a mapping VMA, so dropping
> + * a reference here may not delete it. The view will be marked as
> + * invalid, together with all the VMAs linked to it.
> + */
> + WRITE_ONCE(view->valid, false);
> +
> + /* wait until local faults finish */
> + synchronize_srcu(&fctx->fault_srcu);
> +
> + /*
> + * because the view is marked as invalid, faults will not succeed, so
> + * we don't have to worry about synchronizing invalidations/faults
> + */
> + mirror_clear_view(view, 0, ULONG_MAX, false);
> +
> + view_put(view); /* usage reference */
> +
> + return 0;
> +}
> +
> +static long
> +pidfd_mem_ctrl_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
> +{
> + struct remote_file_context *fctx = file->private_data;
> + void __user *argp = (void __user *)arg;
> + long result = 0;
> +
> + switch (ioctl) {
> + case PIDFD_MEM_MAP: {
> + struct pidfd_mem_map map;
> +
> + result = -EINVAL;
> + if (copy_from_user(&map, argp, sizeof(map)))
> + return result;
> +
> + result = pidfd_mem_map(fctx, &map);
> + break;
> + }
> +
> + case PIDFD_MEM_UNMAP: {
> + struct pidfd_mem_unmap unmap;
> +
> + result = -EINVAL;
> + if (copy_from_user(&unmap, argp, sizeof(unmap)))
> + return result;
> +
> + result = pidfd_mem_unmap(fctx, &unmap);
> + break;
> + }
> +
> + case PIDFD_MEM_LOCK:
> + pidfd_mem_lock(fctx);
> + break;
> +
> + default:
> + pr_debug("%s: ioctl %x not implemented\n", __func__, ioctl);
> + result = -ENOTTY;
> + }
> +
> + return result;
> +}
> +
> +static int pidfd_mem_ctrl_release(struct inode *inode, struct file *file)
> +{
> + struct remote_file_context *fctx = file->private_data;
> +
> + pr_debug("%s: file %p\n", __func__, file);
> +
> + remote_file_context_put(fctx);
> +
> + return 0;
> +}
> +
> +static const struct file_operations pidfd_mem_ctrl_ops = {
> + .owner = THIS_MODULE,
> + .unlocked_ioctl = pidfd_mem_ctrl_ioctl,
> + .compat_ioctl = pidfd_mem_ctrl_ioctl,
<snip>
> +static const struct file_operations pidfd_mem_map_fops = {
> + .owner = THIS_MODULE,
> + .mmap = mirror_dev_mmap,
> + .get_unmapped_area = pidfd_mem_get_unmapped_area,
> + .unlocked_ioctl = pidfd_mem_map_ioctl,
> + .compat_ioctl = pidfd_mem_map_ioctl,
Reading this, it really doesn't feel like a clean API. I think it would
be great if you could investigate whether you can get a more flexible
and cleaner design on top of memfds as suggested elsewhere in the
thread.
Christian
