Re: [PATCH v10 10/26] gunyah: vm_mgr: Introduce basic VM Manager

[Elliot Berman <quic_eberman@xxxxxxxxxxx>]

On 2/21/2023 2:46 AM, Srinivas Kandagatla wrote:
> On 14/02/2023 21:23, Elliot Berman wrote:
> > Gunyah VM manager is a kernel moduel which exposes an interface to
> > Gunyah userspace to load, run, and interact with other Gunyah virtual
> > machines. The interface is a character device at /dev/gunyah.
> >
> > Add a basic VM manager driver. Upcoming patches will add more ioctls
> > into this driver.
> >
> > Co-developed-by: Prakruthi Deepak Heragu <quic_pheragu@xxxxxxxxxxx>
> > Signed-off-by: Prakruthi Deepak Heragu <quic_pheragu@xxxxxxxxxxx>
> > Signed-off-by: Elliot Berman <quic_eberman@xxxxxxxxxxx>
> > ---
> >   .../userspace-api/ioctl/ioctl-number.rst      |   1 +
> >   drivers/virt/gunyah/Makefile                  |   2 +-
> >   drivers/virt/gunyah/rsc_mgr.c                 |  37 +++++-
> >   drivers/virt/gunyah/vm_mgr.c                  | 118 ++++++++++++++++++
> >   drivers/virt/gunyah/vm_mgr.h                  |  22 ++++
> >   include/uapi/linux/gunyah.h                   |  23 ++++
> >   6 files changed, 201 insertions(+), 2 deletions(-)
> >   create mode 100644 drivers/virt/gunyah/vm_mgr.c
> >   create mode 100644 drivers/virt/gunyah/vm_mgr.h
> >   create mode 100644 include/uapi/linux/gunyah.h
> >
> > diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst 
> > b/Documentation/userspace-api/ioctl/ioctl-number.rst
> > index 0a1882e296ae..2513324ae7be 100644
> > --- a/Documentation/userspace-api/ioctl/ioctl-number.rst
> > +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
> > @@ -137,6 +137,7 @@ Code  Seq#    Include 
> > File                                           Comments
> >   'F'   DD     video/sstfb.h                                           
> > conflict!
> >   'G'   00-3F  drivers/misc/sgi-gru/grulib.h                           
> > conflict!
> >   'G'   00-0F  xen/gntalloc.h, xen/gntdev.h                            
> > conflict!
> > +'G'   00-0f  linux/gunyah.h                                          
> > conflict!
> >   'H'   00-7F  linux/hiddev.h                                          
> > conflict!
> >   'H'   00-0F  linux/hidraw.h                                          
> > conflict!
> >   'H'   01     linux/mei.h                                             
> > conflict!
> > diff --git a/drivers/virt/gunyah/Makefile b/drivers/virt/gunyah/Makefile
> > index de29769f2f3f..03951cf82023 100644
> > --- a/drivers/virt/gunyah/Makefile
> > +++ b/drivers/virt/gunyah/Makefile
> > @@ -2,5 +2,5 @@
> >   obj-$(CONFIG_GUNYAH) += gunyah.o
> > -gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o
> > +gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o vm_mgr.o
> >   obj-$(CONFIG_GUNYAH) += gunyah_rsc_mgr.o
> > diff --git a/drivers/virt/gunyah/rsc_mgr.c 
> > b/drivers/virt/gunyah/rsc_mgr.c
> > index 2a47139873a8..73c5a6b7cbbc 100644
> > --- a/drivers/virt/gunyah/rsc_mgr.c
> > +++ b/drivers/virt/gunyah/rsc_mgr.c
> > @@ -16,8 +16,10 @@
> >   #include <linux/completion.h>
> >   #include <linux/gunyah_rsc_mgr.h>
> >   #include <linux/platform_device.h>
> > +#include <linux/miscdevice.h>
> >   #include "rsc_mgr.h"
> > +#include "vm_mgr.h"
> >   #define RM_RPC_API_VERSION_MASK        GENMASK(3, 0)
> >   #define RM_RPC_HEADER_WORDS_MASK    GENMASK(7, 4)
> > @@ -103,6 +105,8 @@ struct gh_rm {
> >       struct kmem_cache *cache;
> >       struct mutex send_lock;
> >       struct blocking_notifier_head nh;
> > +
> > +    struct miscdevice miscdev;
> >   };
> >   static struct gh_rm_connection *gh_rm_alloc_connection(__le32 
> > msg_id, u8 type)
> > @@ -509,6 +513,21 @@ void put_gh_rm(struct gh_rm *rm)
> >   }
> >   EXPORT_SYMBOL_GPL(put_gh_rm);
> > +static long gh_dev_ioctl(struct file *filp, unsigned int cmd, 
> > unsigned long arg)
> > +{
> > +    struct miscdevice *miscdev = filp->private_data;
> > +    struct gh_rm *rm = container_of(miscdev, struct gh_rm, miscdev);
> > +
> > +    return gh_dev_vm_mgr_ioctl(rm, cmd, arg);
> > +}
> > +
> > +static const struct file_operations gh_dev_fops = {
> > +    .owner        = THIS_MODULE,
> > +    .unlocked_ioctl    = gh_dev_ioctl,
> > +    .compat_ioctl    = compat_ptr_ioctl,
> > +    .llseek        = noop_llseek,
> > +};
> > +
> >   static int gh_msgq_platform_probe_direction(struct platform_device 
> > *pdev,
> >                       bool tx, int idx, struct gunyah_resource *ghrsc)
> >   {
> > @@ -567,7 +586,22 @@ static int gh_rm_drv_probe(struct platform_device 
> > *pdev)
> >       rm->msgq_client.rx_callback = gh_rm_msgq_rx_data;
> >       rm->msgq_client.tx_done = gh_rm_msgq_tx_done;
> > -    return gh_msgq_init(&pdev->dev, &rm->msgq, &rm->msgq_client, 
> > &rm->tx_ghrsc, &rm->rx_ghrsc);
> > +    ret = gh_msgq_init(&pdev->dev, &rm->msgq, &rm->msgq_client, 
> > &rm->tx_ghrsc, &rm->rx_ghrsc);
> > +    if (ret)
> > +        goto err_cache;
> > +
> > +    rm->miscdev.name = "gunyah";
> > +    rm->miscdev.minor = MISC_DYNAMIC_MINOR;
> > +    rm->miscdev.fops = &gh_dev_fops;
> > +
> > +    ret = misc_register(&rm->miscdev);
> > +    if (ret)
> > +        goto err_msgq;
> > +
> > +    return 0;
> > +err_msgq:
> > +    mbox_free_channel(gh_msgq_chan(&rm->msgq));
> > +    gh_msgq_remove(&rm->msgq);
> >   err_cache:
> >       kmem_cache_destroy(rm->cache);
> >       return ret;
> > @@ -577,6 +611,7 @@ static int gh_rm_drv_remove(struct platform_device 
> > *pdev)
> >   {
> >       struct gh_rm *rm = platform_get_drvdata(pdev);
> > +    misc_deregister(&rm->miscdev);
> >       mbox_free_channel(gh_msgq_chan(&rm->msgq));
> >       gh_msgq_remove(&rm->msgq);
> >       kmem_cache_destroy(rm->cache);
> > diff --git a/drivers/virt/gunyah/vm_mgr.c b/drivers/virt/gunyah/vm_mgr.c
> > new file mode 100644
> > index 000000000000..fd890a57172e
> > --- /dev/null
> > +++ b/drivers/virt/gunyah/vm_mgr.c
> > @@ -0,0 +1,118 @@
> > +// SPDX-License-Identifier: GPL-2.0-only
> > +/*
> > + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All 
> > rights reserved.
> > + */
> > +
> > +#define pr_fmt(fmt) "gh_vm_mgr: " fmt
> > +
> > +#include <linux/anon_inodes.h>
> > +#include <linux/file.h>
> > +#include <linux/gunyah_rsc_mgr.h>
> > +#include <linux/miscdevice.h>
> > +#include <linux/module.h>
> > +
> > +#include <uapi/linux/gunyah.h>
> > +
> > +#include "vm_mgr.h"
> > +
> > +static void gh_vm_free(struct work_struct *work)
> > +{
> > +    struct gh_vm *ghvm = container_of(work, struct gh_vm, free_work);
> > +    int ret;
> > +
> > +    ret = gh_rm_dealloc_vmid(ghvm->rm, ghvm->vmid);
> > +    if (ret)
> > +        pr_warn("Failed to deallocate vmid: %d\n", ret);
> > +
> > +    put_gh_rm(ghvm->rm);
> > +    kfree(ghvm);
> > +}
> > +
> > +static __must_check struct gh_vm *gh_vm_alloc(struct gh_rm *rm)
> > +{
> > +    struct gh_vm *ghvm;
> > +    int vmid;
> > +
> > +    vmid = gh_rm_alloc_vmid(rm, 0);
> > +    if (vmid < 0)
> > +        return ERR_PTR(vmid);
> > +
> > +    ghvm = kzalloc(sizeof(*ghvm), GFP_KERNEL);
> > +    if (!ghvm) {
> > +        gh_rm_dealloc_vmid(rm, vmid);
> > +        return ERR_PTR(-ENOMEM);
> > +    }
> > +
> > +    get_gh_rm(rm);
> > +
> > +    ghvm->vmid = vmid;
> > +    ghvm->rm = rm;
> > +
> > +    INIT_WORK(&ghvm->free_work, gh_vm_free);
> > +
> > +    return ghvm;
> > +}
> > +
> > +static int gh_vm_release(struct inode *inode, struct file *filp)
> > +{
> > +    struct gh_vm *ghvm = filp->private_data;
> > +
> > +    /* VM will be reset and make RM calls which can interruptible sleep.
> > +     * Defer to a work so this thread can receive signal.
> > +     */
> > +    schedule_work(&ghvm->free_work);
> > +    return 0;
> > +}
> > +
> > +static const struct file_operations gh_vm_fops = {
> > +    .release = gh_vm_release,
>
> > +    .compat_ioctl    = compat_ptr_ioctl,
>
> This line should go with the patch that adds real ioctl

Done.

> > +    .llseek = noop_llseek,
> > +};
> > +
> > +static long gh_dev_ioctl_create_vm(struct gh_rm *rm, unsigned long arg)
> Not sure what is the gain of this multiple levels of redirection.
>
> How about
>
> long gh_dev_create_vm(struct gh_rm *rm, unsigned long arg)
> {
> ...
> }
>
> and rsc_mgr just call it as part of its ioctl call
>
> static long gh_dev_ioctl(struct file *filp, unsigned int cmd, unsigned 
> long arg)
> {
>      struct miscdevice *miscdev = filp->private_data;
>      struct gh_rm *rm = container_of(miscdev, struct gh_rm, miscdev);
>
>      switch (cmd) {
>      case GH_CREATE_VM:
>          return gh_dev_create_vm(rm, arg);
>      default:
>          return -ENOIOCTLCMD;
>      }
> }

I'm anticipating we will add further /dev/gunyah ioctls and I thought it 
would be cleaner to have all that in vm_mgr.c itself.

> > +{
> > +    struct gh_vm *ghvm;
> > +    struct file *file;
> > +    int fd, err;
> > +
> > +    /* arg reserved for future use. */
> > +    if (arg)
> > +        return -EINVAL;
>
> The only code path I see here is via GH_CREATE_VM ioctl which obviously 
> does not take any arguments, so if you are thinking of using the 
> argument for architecture-specific VM flags.  Then this needs to be 
> properly done by making the ABI aware of this.

It is documented in Patch 17 (Document Gunyah VM Manager)

+GH_CREATE_VM
+~~~~~~~~~~~~
+
+Creates a Gunyah VM. The argument is reserved for future use and must be 0.

> As you mentioned zero value arg imply an "unauthenticated VM" type, but 
> this was not properly encoded in the userspace ABI. Why not make it 
> future compatible. How about adding arguments to GH_CREATE_VM and pass 
> the required information correctly.
> Note that once the ABI is accepted then you will not be able to change 
> it, other than adding a new one.

Does this means adding #define GH_VM_DEFAULT_ARG 0 ? I am not sure yet 
what arguments to add here.

The ABI can add new "long" values to GH_CREATE_VM and that wouldn't 
break compatibility with old kernels; old kernels reject it as -EINVAL.

> > +
> > +    ghvm = gh_vm_alloc(rm);
> > +    if (IS_ERR(ghvm))
> > +        return PTR_ERR(ghvm);
> > +
> > +    fd = get_unused_fd_flags(O_CLOEXEC);
> > +    if (fd < 0) {
> > +        err = fd;
> > +        goto err_destroy_vm;
> > +    }
> > +
> > +    file = anon_inode_getfile("gunyah-vm", &gh_vm_fops, ghvm, O_RDWR);
> > +    if (IS_ERR(file)) {
> > +        err = PTR_ERR(file);
> > +        goto err_put_fd;
> > +    }
> > +
> > +    fd_install(fd, file);
> > +
> > +    return fd;
> > +
> > +err_put_fd:
> > +    put_unused_fd(fd);
> > +err_destroy_vm:
> > +    kfree(ghvm);
> > +    return err;
> > +}
> > +
> > +long gh_dev_vm_mgr_ioctl(struct gh_rm *rm, unsigned int cmd, unsigned 
> > long arg)
> > +{
> > +    switch (cmd) {
> > +    case GH_CREATE_VM:
> > +        return gh_dev_ioctl_create_vm(rm, arg);
> > +    default:
> > +        return -ENOIOCTLCMD;
> > +    }
> > +}
> > diff --git a/drivers/virt/gunyah/vm_mgr.h b/drivers/virt/gunyah/vm_mgr.h
> > new file mode 100644
> > index 000000000000..76954da706e9
> > --- /dev/null
> > +++ b/drivers/virt/gunyah/vm_mgr.h
> > @@ -0,0 +1,22 @@
> > +/* SPDX-License-Identifier: GPL-2.0-only */
> > +/*
> > + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All 
> > rights reserved.
> > + */
> > +
> > +#ifndef _GH_PRIV_VM_MGR_H
> > +#define _GH_PRIV_VM_MGR_H
> > +
> > +#include <linux/gunyah_rsc_mgr.h>
> > +
> > +#include <uapi/linux/gunyah.h>
> > +
> > +long gh_dev_vm_mgr_ioctl(struct gh_rm *rm, unsigned int cmd, unsigned 
> > long arg);
> > +
> > +struct gh_vm {
> > +    u16 vmid;
> > +    struct gh_rm *rm;
> > +
> > +    struct work_struct free_work;
> > +};
> > +
> > +#endif
> > diff --git a/include/uapi/linux/gunyah.h b/include/uapi/linux/gunyah.h
> > new file mode 100644
> > index 000000000000..10ba32d2b0a6
> > --- /dev/null
> > +++ b/include/uapi/linux/gunyah.h
> > @@ -0,0 +1,23 @@
> > +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
> > +/*
> > + * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All 
> > rights reserved.
> > + */
> > +
> > +#ifndef _UAPI_LINUX_GUNYAH
> > +#define _UAPI_LINUX_GUNYAH
> > +
> > +/*
> > + * Userspace interface for /dev/gunyah - gunyah based virtual machine
> > + */
> > +
> > +#include <linux/types.h>
> > +#include <linux/ioctl.h>
> > +
> > +#define GH_IOCTL_TYPE            'G'
> > +
> > +/*
> > + * ioctls for /dev/gunyah fds:
> > + */
> > +#define GH_CREATE_VM            _IO(GH_IOCTL_TYPE, 0x0) /* Returns a 
> > Gunyah VM fd */
>
> Can HLOS forcefully destroy a VM?
> If so should we have a corresponding DESTROY IOCTL?

It can forcefully destroy unauthenticated and protected virtual 
machines. I don't have a userspace usecase for a DESTROY ioctl yet, 
maybe this can be added later? By the way, the VM is forcefully 
destroyed when VM refcount is dropped to 0 (close(vm_fd) and any other 
relevant file descriptors).

- Elliot