Design for vGPU Driver:
Main purpose of vGPU driver is to provide a common interface for vGPU
management that can be used by differnt GPU drivers.
This module would provide a generic interface to create the device, add
it to vGPU bus, add device to IOMMU group and then add it to vfio group.
High Level block diagram:
+--------------+ vgpu_register_driver()+---------------+
| __init() +------------------------->+ |
| | | |
| +<-------------------------+ vgpu.ko |
| vfio_vgpu.ko | probe()/remove() | |
| | +---------+ +---------+
+--------------+ | +-------+-------+ |
| ^ |
| callback | |
| +-------+--------+ |
| |vgpu_register_device() |
| | | |
+---^-----+-----+ +-----+------+-+
| nvidia.ko | | i915.ko |
| | | |
+-----------+ +------------+
vGPU driver provides two types of registration interfaces:
1. Registration interface for vGPU bus driver:
/**
* struct vgpu_driver - vGPU device driver
* @name: driver name
* @probe: called when new device created
* @remove: called when device removed
* @driver: device driver structure
*
**/
struct vgpu_driver {
const char *name;
int (*probe) (struct device *dev);
void (*remove) (struct device *dev);
struct device_driver driver;
};
int vgpu_register_driver(struct vgpu_driver *drv, struct module *owner);
void vgpu_unregister_driver(struct vgpu_driver *drv);
VFIO bus driver for vgpu, should use this interface to register with
vGPU driver. With this, VFIO bus driver for vGPU devices is responsible
to add vGPU device to VFIO group.
2. GPU driver interface
/**
* struct gpu_device_ops - Structure to be registered for each physical
GPU to
* register the device to vgpu module.
*
* @owner: The module owner.
* @vgpu_supported_config: Called to get information about supported
* vgpu types.
* @dev : pci device structure of physical GPU.
* @config: should return string listing supported
* config
* Returns integer: success (0) or error (< 0)
* @vgpu_create: Called to allocate basic resouces in graphics
* driver for a particular vgpu.
* @dev: physical pci device structure on which vgpu
* should be created
* @uuid: uuid for which VM it is intended to
* @instance: vgpu instance in that VM
* @vgpu_id: This represents the type of vgpu to be
* created
* Returns integer: success (0) or error (< 0)
* @vgpu_destroy: Called to free resources in graphics driver for
* a vgpu instance of that VM.
* @dev: physical pci device structure to which
* this vgpu points to.
* @uuid: uuid for which the vgpu belongs to.
* @instance: vgpu instance in that VM
* Returns integer: success (0) or error (< 0)
* If VM is running and vgpu_destroy is called that
* means the vGPU is being hotunpluged. Return error
* if VM is running and graphics driver doesn't
* support vgpu hotplug.
* @vgpu_start: Called to do initiate vGPU initialization
* process in graphics driver when VM boots before
* qemu starts.
* @uuid: UUID which is booting.
* Returns integer: success (0) or error (< 0)
* @vgpu_shutdown: Called to teardown vGPU related resources for
* the VM
* @uuid: UUID which is shutting down .
* Returns integer: success (0) or error (< 0)
* @read: Read emulation callback
* @vdev: vgpu device structure
* @buf: read buffer
* @count: number bytes to read
* @address_space: specifies for which address space
* the request is: pci_config_space, IO register
* space or MMIO space.
* Retuns number on bytes read on success or error.
* @write: Write emulation callback
* @vdev: vgpu device structure
* @buf: write buffer
* @count: number bytes to be written
* @address_space: specifies for which address space
* the request is: pci_config_space, IO register
* space or MMIO space.
* Retuns number on bytes written on success or
error.
* @vgpu_set_irqs: Called to send about interrupts configuration
* information that qemu set.
* @vdev: vgpu device structure
* @flags, index, start, count and *data : same as
* that of struct vfio_irq_set of
* VFIO_DEVICE_SET_IRQS API.
*
* Physical GPU that support vGPU should be register with vgpu module with
* gpu_device_ops structure.
*/
struct gpu_device_ops {
struct module *owner;
int (*vgpu_supported_config)(struct pci_dev *dev,
char *config);
int (*vgpu_create)(struct pci_dev *dev, uuid_le uuid,
uint32_t instance, uint32_t vgpu_id);
int (*vgpu_destroy)(struct pci_dev *dev, uuid_le uuid,
uint32_t instance);
int (*vgpu_start)(uuid_le uuid);
int (*vgpu_shutdown)(uuid_le uuid);
ssize_t (*read) (struct vgpu_device *vdev, char *buf,
size_t count, uint32_t address_space,
loff_t pos);
ssize_t (*write)(struct vgpu_device *vdev, char *buf,
size_t count, uint32_t address_space,
loff_t pos);
int (*vgpu_set_irqs)(struct vgpu_device *vdev,
uint32_t flags, unsigned index,
unsigned start, unsigned count,
void *data);
};
int vgpu_register_device(struct pci_dev *dev, const struct
gpu_device_ops *ops);
void vgpu_unregister_device(struct pci_dev *dev);
This registration interface should be used by GPU drivers to register
each physical device to vGPU driver.
This patch set is to review major APIs and design structure before
continuing development and don't consider as final patch for review.
These registration interfaces would evolve as the development proceed.
Regards,
Kirti
Signed-off-by: Kirti Wankhede <kwankhede@xxxxxxxxxx>
Signed-off-by: Neo Jia <cjia@xxxxxxxxxx>
---
---
drivers/vgpu/vgpu-driver.c | 137 ++++++++++++++
drivers/vgpu/vgpu_dev.c | 421
++++++++++++++++++++++++++++++++++++++++++++
include/linux/vgpu.h | 189 ++++++++++++++++++++
3 files changed, 747 insertions(+), 0 deletions(-)
create mode 100644 drivers/vgpu/vgpu-driver.c
create mode 100644 drivers/vgpu/vgpu_dev.c
create mode 100644 include/linux/vgpu.h
diff --git a/drivers/vgpu/vgpu-driver.c b/drivers/vgpu/vgpu-driver.c
new file mode 100644
index 0000000..6b62f19
--- /dev/null
+++ b/drivers/vgpu/vgpu-driver.c
@@ -0,0 +1,137 @@
+/*
+ * VGPU driver
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ * Author: Neo Jia <cjia@xxxxxxxxxx>
+ * Kirti Wankhede <kwankhede@xxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/vfio.h>
+#include <linux/iommu.h>
+#include <linux/sysfs.h>
+#include <linux/ctype.h>
+#include <linux/vgpu.h>
+
+#include "vgpu_private.h"
+
+static int vgpu_device_attach_iommu(struct vgpu_device *vgpu_dev)
+{
+ int retval = 0;
+ struct iommu_group *group = NULL;
+
+ group = iommu_group_alloc();
+ if (IS_ERR(group)) {
+ printk(KERN_ERR "VGPU: failed to allocate group!\n");
+ return PTR_ERR(group);
+ }
+
+ retval = iommu_group_add_device(group, &vgpu_dev->dev);
+ if (retval) {
+ printk(KERN_ERR "VGPU: failed to add dev to group!\n");
+ iommu_group_put(group);
+ return retval;
+ }
+
+ vgpu_dev->group = group;
+
+ printk(KERN_INFO "VGPU: group_id = %d \n", iommu_group_id(group));
+ return retval;
+}
+
+static void vgpu_device_detach_iommu(struct vgpu_device *vgpu_dev)
+{
+ iommu_group_put(vgpu_dev->dev.iommu_group);
+ iommu_group_remove_device(&vgpu_dev->dev);
+ printk(KERN_INFO "VGPU: detaching iommu \n");
+}
+
+static int vgpu_device_probe(struct device *dev)
+{
+ struct vgpu_driver *drv = to_vgpu_driver(dev->driver);
+ struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
+ int status = 0;
+
+ status = vgpu_device_attach_iommu(vgpu_dev);
+ if (status) {
+ printk(KERN_ERR "Failed to attach IOMMU\n");
+ return status;
+ }
+
+ if (drv && drv->probe) {
+ status = drv->probe(dev);
+ }
+
+ return status;
+}
+
+static int vgpu_device_remove(struct device *dev)
+{
+ struct vgpu_driver *drv = to_vgpu_driver(dev->driver);
+ struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
+ int status = 0;
+
+ if (drv && drv->remove) {
+ drv->remove(dev);
+ }
+
+ vgpu_device_detach_iommu(vgpu_dev);
+
+ return status;
+}
+
+struct bus_type vgpu_bus_type = {
+ .name = "vgpu",
+ .probe = vgpu_device_probe,
+ .remove = vgpu_device_remove,
+};
+EXPORT_SYMBOL_GPL(vgpu_bus_type);
+
+/**
+ * vgpu_register_driver - register a new vGPU driver
+ * @drv: the driver to register
+ * @owner: owner module of driver ro register
+ *
+ * Returns a negative value on error, otherwise 0.
+ */
+int vgpu_register_driver(struct vgpu_driver *drv, struct module *owner)
+{
+ /* initialize common driver fields */
+ drv->driver.name = drv->name;
+ drv->driver.bus = &vgpu_bus_type;
+ drv->driver.owner = owner;
+
+ /* register with core */
+ return driver_register(&drv->driver);
+}
+EXPORT_SYMBOL(vgpu_register_driver);
+
+/**
+ * vgpu_unregister_driver - unregister vGPU driver
+ * @drv: the driver to unregister
+ *
+ */
+void vgpu_unregister_driver(struct vgpu_driver *drv)
+{
+ driver_unregister(&drv->driver);
+}
+EXPORT_SYMBOL(vgpu_unregister_driver);
+
+int vgpu_bus_register(void)
+{
+ return bus_register(&vgpu_bus_type);
+}
+
+void vgpu_bus_unregister(void)
+{
+ bus_unregister(&vgpu_bus_type);
+}
+
diff --git a/drivers/vgpu/vgpu_dev.c b/drivers/vgpu/vgpu_dev.c
new file mode 100644
index 0000000..e5a92a1
--- /dev/null
+++ b/drivers/vgpu/vgpu_dev.c
@@ -0,0 +1,421 @@
+/*
+ * VGPU Core Driver
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ * Author: Neo Jia <cjia@xxxxxxxxxx>
+ * Kirti Wankhede <kwankhede@xxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/cdev.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/uuid.h>
+#include <linux/vfio.h>
+#include <linux/iommu.h>
+#include <linux/sysfs.h>
+#include <linux/ctype.h>
+#include <linux/vgpu.h>
+
+#include "vgpu_private.h"
+
+#define DRIVER_VERSION "0.1"
+#define DRIVER_AUTHOR "NVIDIA Corporation"
+#define DRIVER_DESC "VGPU Core Driver"
+
+/*
+ * #defines
+ */
+
+#define VGPU_CLASS_NAME "vgpu"
+
+/*
+ * Global Structures
+ */
+
+static struct vgpu {
+ struct list_head vgpu_devices_list;
+ struct mutex vgpu_devices_lock;
+ struct list_head gpu_devices_list;
+ struct mutex gpu_devices_lock;
+} vgpu;
+
+
+static struct class vgpu_class;
+
+/*
+ * Function prototypes
+ */
+
+unsigned int vgpu_poll(struct file *file, poll_table *wait);
+long vgpu_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned
long i_arg);
+int vgpu_mmap(struct file *file, struct vm_area_struct *vma);
+
+int vgpu_open(struct inode *inode, struct file *file);
+int vgpu_close(struct inode *inode, struct file *file);
+ssize_t vgpu_read(struct file *file, char __user * buf,
+ size_t len, loff_t * ppos);
+ssize_t vgpu_write(struct file *file, const char __user *data,
+ size_t len, loff_t *ppos);
+
+/*
+ * Functions
+ */
+
+struct vgpu_device *get_vgpu_device_from_group(struct iommu_group *group)
+{
+ struct vgpu_device *vdev = NULL;
+
+ mutex_lock(&vgpu.vgpu_devices_lock);
+ list_for_each_entry(vdev, &vgpu.vgpu_devices_list, list) {
+ if (vdev->group) {
+ if (iommu_group_id(vdev->group) == iommu_group_id(group)) {
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+ return vdev;
+ }
+ }
+ }
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+ return NULL;
+}
+
+EXPORT_SYMBOL_GPL(get_vgpu_device_from_group);
+
+int vgpu_register_device(struct pci_dev *dev, const struct
gpu_device_ops *ops)
+{
+ int ret = 0;
+ struct gpu_device *gpu_dev, *tmp;
+
+ if (!dev)
+ return -EINVAL;
+
+ gpu_dev = kzalloc(sizeof(*gpu_dev), GFP_KERNEL);
+ if (!gpu_dev)
+ return -ENOMEM;
+
+ gpu_dev->dev = dev;
+ gpu_dev->ops = ops;
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+
+ /* Check for duplicates */
+ list_for_each_entry(tmp, &vgpu.gpu_devices_list, gpu_next) {
+ if (tmp->dev == dev) {
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ kfree(gpu_dev);
+ return -EINVAL;
+ }
+ }
+
+ ret = vgpu_create_pci_device_files(dev);
+ if (ret) {
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ kfree(gpu_dev);
+ return ret;
+ }
+ list_add(&gpu_dev->gpu_next, &vgpu.gpu_devices_list);
+
+ printk(KERN_INFO "VGPU: Registered dev 0x%x 0x%x, class 0x%x\n",
dev->vendor, dev->device, dev->class);
+ mutex_unlock(&vgpu.gpu_devices_lock);
+
+ return 0;
+}
+EXPORT_SYMBOL(vgpu_register_device);
+
+void vgpu_unregister_device(struct pci_dev *dev)
+{
+ struct gpu_device *gpu_dev;
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+ list_for_each_entry(gpu_dev, &vgpu.gpu_devices_list, gpu_next) {
+ if (gpu_dev->dev == dev) {
+ printk(KERN_INFO "VGPU: Unregistered dev 0x%x 0x%x, class
0x%x\n", dev->vendor, dev->device, dev->class);
+ vgpu_remove_pci_device_files(dev);
+ list_del(&gpu_dev->gpu_next);
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ kfree(gpu_dev);
+ return;
+ }
+ }
+ mutex_unlock(&vgpu.gpu_devices_lock);
+}
+EXPORT_SYMBOL(vgpu_unregister_device);
+
+/*
+ * Helper Functions
+ */
+
+static struct vgpu_device *vgpu_device_alloc(uuid_le uuid, int
instance, char *name)
+{
+ struct vgpu_device *vgpu_dev = NULL;
+
+ vgpu_dev = kzalloc(sizeof(*vgpu_dev), GFP_KERNEL);
+ if (!vgpu_dev)
+ return ERR_PTR(-ENOMEM);
+
+ kref_init(&vgpu_dev->kref);
+ memcpy(&vgpu_dev->uuid, &uuid, sizeof(uuid_le));
+ vgpu_dev->vgpu_instance = instance;
+ strcpy(vgpu_dev->dev_name, name);
+
+ mutex_lock(&vgpu.vgpu_devices_lock);
+ list_add(&vgpu_dev->list, &vgpu.vgpu_devices_list);
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+
+ return vgpu_dev;
+}
+
+static void vgpu_device_free(struct vgpu_device *vgpu_dev)
+{
+ if (vgpu_dev) {
+ mutex_lock(&vgpu.vgpu_devices_lock);
+ list_del(&vgpu_dev->list);
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+ kfree(vgpu_dev);
+ }
+ return;
+}
+
+struct vgpu_device *vgpu_drv_get_vgpu_device_by_uuid(uuid_le uuid, int
instance)
+{
+ struct vgpu_device *vdev = NULL;
+
+ mutex_lock(&vgpu.vgpu_devices_lock);
+ list_for_each_entry(vdev, &vgpu.vgpu_devices_list, list) {
+ if ((uuid_le_cmp(vdev->uuid, uuid) == 0) &&
+ (vdev->vgpu_instance == instance)) {
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+ return vdev;
+ }
+ }
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+ return NULL;
+}
+
+static void vgpu_device_release(struct device *dev)
+{
+ struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
+ vgpu_device_free(vgpu_dev);
+}
+
+int create_vgpu_device(struct pci_dev *pdev, uuid_le uuid, uint32_t
instance, uint32_t vgpu_id)
+{
+ char name[64];
+ int numChar = 0;
+ int retval = 0;
+ struct vgpu_device *vgpu_dev = NULL;
+ struct gpu_device *gpu_dev;
+
+ printk(KERN_INFO "VGPU: %s: device ", __FUNCTION__);
+
+ numChar = sprintf(name, "%pUb-%d", uuid.b, instance);
+ name[numChar] = '\0';
+
+ vgpu_dev = vgpu_device_alloc(uuid, instance, name);
+ if (IS_ERR(vgpu_dev)) {
+ return PTR_ERR(vgpu_dev);
+ }
+
+ vgpu_dev->vgpu_id = vgpu_id;
+ vgpu_dev->dev.parent = NULL;
+ vgpu_dev->dev.bus = &vgpu_bus_type;
+ vgpu_dev->dev.release = vgpu_device_release;
+ dev_set_name(&vgpu_dev->dev, "%s", name);
+
+ retval = device_register(&vgpu_dev->dev);
+ if (retval)
+ goto create_failed1;
+
+ printk(KERN_INFO "UUID %pUb \n", vgpu_dev->uuid.b);
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+ list_for_each_entry(gpu_dev, &vgpu.gpu_devices_list, gpu_next) {
+ if (gpu_dev->dev == pdev) {
+ vgpu_dev->gpu_dev = gpu_dev;
+ if (gpu_dev->ops->vgpu_create) {
+ retval = gpu_dev->ops->vgpu_create(pdev, vgpu_dev->uuid,
+ instance, vgpu_id);
+ if (retval) {
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ goto create_failed2;
+ }
+ }
+ break;
+ }
+ }
+ if (!vgpu_dev->gpu_dev) {
+ retval = -EINVAL;
+ goto create_failed2;
+ }
+
+ mutex_unlock(&vgpu.gpu_devices_lock);
+
+ return retval;
+
+create_failed2:
+ device_unregister(&vgpu_dev->dev);
+
+create_failed1:
+ vgpu_device_free(vgpu_dev);
+
+ return retval;
+}
+
+void destroy_vgpu_device(struct vgpu_device *vgpu_dev)
+{
+ printk(KERN_INFO "VGPU: destroying device %s ", vgpu_dev->dev_name);
+ if (vgpu_dev->gpu_dev->ops->vgpu_destroy) {
+ int retval = 0;
+ retval =
vgpu_dev->gpu_dev->ops->vgpu_destroy(vgpu_dev->gpu_dev->dev,
+ vgpu_dev->uuid,
+ vgpu_dev->vgpu_instance);
+ /* if vendor driver doesn't return success that means vendor driver
doesn't
+ * support hot-unplug */
+ if (retval)
+ return;
+ }
+
+ device_unregister(&vgpu_dev->dev);
+}
+
+void destroy_vgpu_device_by_uuid(uuid_le uuid, int instance)
+{
+ struct vgpu_device *vdev, *vgpu_dev = NULL;
+
+ mutex_lock(&vgpu.vgpu_devices_lock);
+
+ // search VGPU device
+ list_for_each_entry(vdev, &vgpu.vgpu_devices_list, list) {
+ if ((uuid_le_cmp(vdev->uuid, uuid) == 0) &&
+ (vdev->vgpu_instance == instance)) {
+ vgpu_dev = vdev;
+ break;
+ }
+ }
+
+ mutex_unlock(&vgpu.vgpu_devices_lock);
+ if (vgpu_dev)
+ destroy_vgpu_device(vgpu_dev);
+}
+
+void get_vgpu_supported_types(struct device *dev, char *str)
+{
+ struct gpu_device *gpu_dev;
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+ list_for_each_entry(gpu_dev, &vgpu.gpu_devices_list, gpu_next) {
+ if (&gpu_dev->dev->dev == dev) {
+ if (gpu_dev->ops->vgpu_supported_config)
+ gpu_dev->ops->vgpu_supported_config(gpu_dev->dev, str);
+ break;
+ }
+ }
+ mutex_unlock(&vgpu.gpu_devices_lock);
+}
+
+int vgpu_start_callback(struct vgpu_device *vgpu_dev)
+{
+ int ret = 0;
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+ if (vgpu_dev->gpu_dev->ops->vgpu_start)
+ ret = vgpu_dev->gpu_dev->ops->vgpu_start(vgpu_dev->uuid);
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ return ret;
+}
+
+int vgpu_shutdown_callback(struct vgpu_device *vgpu_dev)
+{
+ int ret = 0;
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+ if (vgpu_dev->gpu_dev->ops->vgpu_shutdown)
+ ret = vgpu_dev->gpu_dev->ops->vgpu_shutdown(vgpu_dev->uuid);
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ return ret;
+}
+
+int vgpu_set_irqs_callback(struct vgpu_device *vgpu_dev, uint32_t flags,
+ unsigned index, unsigned start, unsigned count,
+ void *data)
+{
+ int ret = 0;
+
+ mutex_lock(&vgpu.gpu_devices_lock);
+ if (vgpu_dev->gpu_dev->ops->vgpu_set_irqs)
+ ret = vgpu_dev->gpu_dev->ops->vgpu_set_irqs(vgpu_dev,
flags,
+ index, start,
count, data);
+ mutex_unlock(&vgpu.gpu_devices_lock);
+ return ret;
+}
+
+char *vgpu_devnode(struct device *dev, umode_t *mode)
+{
+ return kasprintf(GFP_KERNEL, "vgpu/%s", dev_name(dev));
+}
+
+static void release_vgpubus_dev(struct device *dev)
+{
+ struct vgpu_device *vgpu_dev = to_vgpu_device(dev);
+ destroy_vgpu_device(vgpu_dev);
+}
+
+static struct class vgpu_class = {
+ .name = VGPU_CLASS_NAME,
+ .owner = THIS_MODULE,
+ .class_attrs = vgpu_class_attrs,
+ .dev_groups = vgpu_dev_groups,
+ .devnode = vgpu_devnode,
+ .dev_release = release_vgpubus_dev,
+};
+
+static int __init vgpu_init(void)
+{
+ int rc = 0;
+
+ memset(&vgpu, 0 , sizeof(vgpu));
+
+ mutex_init(&vgpu.vgpu_devices_lock);
+ INIT_LIST_HEAD(&vgpu.vgpu_devices_list);
+ mutex_init(&vgpu.gpu_devices_lock);
+ INIT_LIST_HEAD(&vgpu.gpu_devices_list);
+
+ rc = class_register(&vgpu_class);
+ if (rc < 0) {
+ printk(KERN_ERR "Error: failed to register vgpu class\n");
+ goto failed1;
+ }
+
+ rc = vgpu_bus_register();
+ if (rc < 0) {
+ printk(KERN_ERR "Error: failed to register vgpu bus\n");
+ class_unregister(&vgpu_class);
+ }
+
+failed1:
+ return rc;
+}
+
+static void __exit vgpu_exit(void)
+{
+ vgpu_bus_unregister();
+ class_unregister(&vgpu_class);
+}
+
+module_init(vgpu_init)
+module_exit(vgpu_exit)
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/include/linux/vgpu.h b/include/linux/vgpu.h
new file mode 100644
index 0000000..8595df5
--- /dev/null
+++ b/include/linux/vgpu.h
@@ -0,0 +1,189 @@
+/*
+ * VGPU definition
+ *
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ * Author:
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef VGPU_H
+#define VGPU_H
+
+// Common Data structures
+
+struct pci_bar_info {
+ uint64_t start;
+ uint64_t end;
+ int flags;
+};
+
+enum vgpu_emul_space_e {
+ vgpu_emul_space_config = 0, /*!< PCI configuration space */
+ vgpu_emul_space_io = 1, /*!< I/O register space */
+ vgpu_emul_space_mmio = 2 /*!< Memory-mapped I/O space */
+};
+
+struct gpu_device;
+
+/*
+ * VGPU device
+ */
+struct vgpu_device {
+ struct kref kref;
+ struct device dev;
+ struct gpu_device *gpu_dev;
+ struct iommu_group *group;
+#define DEVICE_NAME_LEN (64)
+ char dev_name[DEVICE_NAME_LEN];
+ uuid_le uuid;
+ uint32_t vgpu_instance;
+ uint32_t vgpu_id;
+ char config_space[0x100]; // 4KB PCI cfg space
+ struct pci_bar_info bar[VFIO_PCI_NUM_REGIONS];
+ struct device_attribute *dev_attr_vgpu_status;
+ int vgpu_device_status;
+
+ void *driver_data;
+
+ struct list_head list;
+};
+
+
+/**
+ * struct gpu_device_ops - Structure to be registered for each physical
GPU to
+ * register the device to vgpu module.
+ *
+ * @owner: The module owner.
+ * @vgpu_supported_config: Called to get information about supported
vgpu types.
+ * @dev : pci device structure of physical GPU.
+ * @config: should return string listing supported config
+ * Returns integer: success (0) or error (< 0)
+ * @vgpu_create: Called to allocate basic resouces in graphics
+ * driver for a particular vgpu.
+ * @dev: physical pci device structure on which vgpu
+ * should be created
+ * @uuid: uuid for which VM it is intended to
+ * @instance: vgpu instance in that VM
+ * @vgpu_id: This represents the type of vgpu to be
+ * created
+ * Returns integer: success (0) or error (< 0)
+ * @vgpu_destroy: Called to free resources in graphics driver for
+ * a vgpu instance of that VM.
+ * @dev: physical pci device structure to which
+ * this vgpu points to.
+ * @uuid: uuid for which the vgpu belongs to.
+ * @instance: vgpu instance in that VM
+ * Returns integer: success (0) or error (< 0)
+ * If VM is running and vgpu_destroy is called that
+ * means the vGPU is being hotunpluged. Return error
+ * if VM is running and graphics driver doesn't
+ * support vgpu hotplug.
+ * @vgpu_start: Called to do initiate vGPU initialization
+ * process in graphics driver when VM boots before
+ * qemu starts.
+ * @uuid: UUID which is booting.
+ * Returns integer: success (0) or error (< 0)
+ * @vgpu_shutdown: Called to teardown vGPU related resources for
+ * the VM
+ * @uuid: UUID which is shutting down .
+ * Returns integer: success (0) or error (< 0)
+ * @read: Read emulation callback
+ * @vdev: vgpu device structure
+ * @buf: read buffer
+ * @count: number bytes to read
+ * @address_space: specifies for which address space
+ * the request is: pci_config_space, IO register
+ * space or MMIO space.
+ * Retuns number on bytes read on success or error.
+ * @write: Write emulation callback
+ * @vdev: vgpu device structure
+ * @buf: write buffer
+ * @count: number bytes to be written
+ * @address_space: specifies for which address space
+ * the request is: pci_config_space, IO register
+ * space or MMIO space.
+ * Retuns number on bytes written on success or error.
+ * @vgpu_set_irqs: Called to send about interrupts configuration
+ * information that qemu set.
+ * @vdev: vgpu device structure
+ * @flags, index, start, count and *data : same as
+ * that of struct vfio_irq_set of
+ * VFIO_DEVICE_SET_IRQS API.
+ *
+ * Physical GPU that support vGPU should be register with vgpu module with
+ * gpu_device_ops structure.
+ */
+
+struct gpu_device_ops {
+ struct module *owner;
+ int (*vgpu_supported_config)(struct pci_dev *dev, char *config);
+ int (*vgpu_create)(struct pci_dev *dev, uuid_le uuid,
+ uint32_t instance, uint32_t vgpu_id);
+ int (*vgpu_destroy)(struct pci_dev *dev, uuid_le uuid,
+ uint32_t instance);
+ int (*vgpu_start)(uuid_le uuid);
+ int (*vgpu_shutdown)(uuid_le uuid);
+ ssize_t (*read) (struct vgpu_device *vdev, char *buf, size_t count,
+ uint32_t address_space, loff_t pos);
+ ssize_t (*write)(struct vgpu_device *vdev, char *buf, size_t count,
+ uint32_t address_space,loff_t pos);
+ int (*vgpu_set_irqs)(struct vgpu_device *vdev, uint32_t flags,
+ unsigned index, unsigned start, unsigned count,
+ void *data);
+
+};
+
+/*
+ * Physical GPU
+ */
+struct gpu_device {
+ struct pci_dev *dev;
+ const struct gpu_device_ops *ops;
+ struct list_head gpu_next;
+};
+
+/**
+ * struct vgpu_driver - vGPU device driver
+ * @name: driver name
+ * @probe: called when new device created
+ * @remove: called when device removed
+ * @driver: device driver structure
+ *
+ **/
+struct vgpu_driver {
+ const char *name;
+ int (*probe) (struct device *dev);
+ void (*remove) (struct device *dev);
+ struct device_driver driver;
+};
+
+static inline struct vgpu_driver *to_vgpu_driver(struct device_driver
*drv)
+{
+ return drv ? container_of(drv, struct vgpu_driver, driver) : NULL;
+}
+
+static inline struct vgpu_device *to_vgpu_device(struct device *dev)
+{
+ return dev ? container_of(dev, struct vgpu_device, dev) : NULL;
+}
+
+extern struct bus_type vgpu_bus_type;
+
+#define dev_is_vgpu(d) ((d)->bus == &vgpu_bus_type)
+
+extern int vgpu_register_device(struct pci_dev *dev, const struct
gpu_device_ops *ops);
+extern void vgpu_unregister_device(struct pci_dev *dev);
+
+extern int vgpu_register_driver(struct vgpu_driver *drv, struct module
*owner);
+extern void vgpu_unregister_driver(struct vgpu_driver *drv);
+
+extern int vgpu_map_virtual_bar(uint64_t virt_bar_addr, uint64_t
phys_bar_addr, uint32_t len, uint32_t flags);
+extern int vgpu_dma_do_translate(dma_addr_t * gfn_buffer, uint32_t count);
+
+struct vgpu_device *get_vgpu_device_from_group(struct iommu_group *group);
+
+#endif /* VGPU_H */
+
