[PATCH RFC 1/1] QEMU vhost-scsi: add support for VHOST_SET_VRING_WORKER

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



This patch adds support for the proposed ioctl that allows userspace
to create virtqueue workers. For vhost-scsi you can set virtqueue_workers
to:

 0: default behavior where we have 1 worker for all vqs.
-1: create a worker per vq
>0: create N workers and allow the vqs to share them by assigning a
vq a worker by just doing round robin.

TODO:
- Allow sharing workers across devices.
- Bind to specific CPUs. Commands like "virsh emulatorpin" allow us
to set the group of vhost threads to different CPUs. But we can also
set a specific vq's worker to run on a CPU.
- I'm handling old kernel by just checking for EPERM. Does this require
a feature?

Signed-off-by: Mike Christie <michael.christie@xxxxxxxxxx>
---
 hw/scsi/vhost-scsi.c                         | 85 +++++++++++++++++++-
 hw/virtio/vhost-backend.c                    |  8 ++
 include/hw/virtio/vhost-backend.h            |  4 +
 include/hw/virtio/virtio-scsi.h              |  1 +
 include/standard-headers/linux/vhost_types.h |  9 +++
 linux-headers/linux/vhost.h                  |  7 ++
 6 files changed, 111 insertions(+), 3 deletions(-)

diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 4d70fa036bbe..9e3653d158c3 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -163,6 +163,76 @@ static const VMStateDescription vmstate_virtio_vhost_scsi = {
     .pre_save = vhost_scsi_pre_save,
 };
 
+static int vhost_scsi_set_workers(VHostSCSICommon *vsc, int vq_workers)
+{
+    struct vhost_dev *dev = &vsc->dev;
+    int worker_index = 0, num_workers = 0;
+    struct vhost_vring_worker w;
+    pid_t *workers = NULL;
+    int i, ret;
+
+    if (vq_workers < -1)
+        return -EINVAL;
+
+   if (vq_workers > 0) {
+       if (vq_workers > dev->nvqs)
+           vq_workers = dev->nvqs;
+
+       workers = g_malloc0(vq_workers * sizeof(pid_t));
+    }
+
+    w.pid = -1;
+    for (i = 0; i < dev->nvqs; i++) {
+        w.index = i;
+
+        switch (vq_workers) {
+        case -1:
+             /*
+              * ctl/evt share the first worker since it will be rare for them
+              * to send cmds while IO is running. The rest of the vqs get their
+              * own worker.
+              */
+            if (i > VHOST_SCSI_VQ_NUM_FIXED)
+                w.pid = -1;
+            break;
+	case 0:
+            /* All vqs share 1 worker. Pass back the pid we got the first run */
+            break;
+        default:
+            /* Each worker handles N vqs. */
+            if (num_workers == vq_workers) {
+                w.pid = workers[worker_index];
+
+                worker_index++;
+                if (worker_index == vq_workers)
+                    worker_index = 0;
+            } else {
+                w.pid = -1;
+            }
+
+            break;
+        }
+
+        ret = dev->vhost_ops->vhost_set_vring_worker(dev, &w);
+        /* Ignore for now. Add feature in final patch */
+        if (ret == -EPERM) {
+            ret = 0;
+            goto free_workers;
+        } else if (ret)
+            goto free_workers;
+
+        if (vq_workers > 0 && num_workers < vq_workers) {
+            workers[num_workers] = w.pid;
+            num_workers++;
+        }
+    }
+
+free_workers:
+    if (workers)
+        g_free(workers);
+    return ret;
+}
+
 static void vhost_scsi_realize(DeviceState *dev, Error **errp)
 {
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
@@ -226,6 +296,13 @@ static void vhost_scsi_realize(DeviceState *dev, Error **errp)
         goto free_vqs;
     }
 
+    ret = vhost_scsi_set_workers(vsc, vs->conf.virtqueue_workers);
+    if (ret < 0) {
+        error_setg(errp, "vhost-scsi: vhost worker setup failed: %s",
+                   strerror(-ret));
+        goto free_vqs;
+    }
+
     /* At present, channel and lun both are 0 for bootable vhost-scsi disk */
     vsc->channel = 0;
     vsc->lun = 0;
@@ -271,18 +348,20 @@ static Property vhost_scsi_properties[] = {
     DEFINE_PROP_STRING("wwpn", VirtIOSCSICommon, conf.wwpn),
     DEFINE_PROP_UINT32("boot_tpgt", VirtIOSCSICommon, conf.boot_tpgt, 0),
     DEFINE_PROP_UINT32("num_queues", VirtIOSCSICommon, conf.num_queues,
-                       VIRTIO_SCSI_AUTO_NUM_QUEUES),
+                       8),
     DEFINE_PROP_UINT32("virtqueue_size", VirtIOSCSICommon, conf.virtqueue_size,
-                       128),
+                       1024),
     DEFINE_PROP_BOOL("seg_max_adjust", VirtIOSCSICommon, conf.seg_max_adjust,
                       true),
     DEFINE_PROP_UINT32("max_sectors", VirtIOSCSICommon, conf.max_sectors,
                        0xFFFF),
-    DEFINE_PROP_UINT32("cmd_per_lun", VirtIOSCSICommon, conf.cmd_per_lun, 128),
+    DEFINE_PROP_UINT32("cmd_per_lun", VirtIOSCSICommon, conf.cmd_per_lun, 1024),
     DEFINE_PROP_BIT64("t10_pi", VHostSCSICommon, host_features,
                                                  VIRTIO_SCSI_F_T10_PI,
                                                  false),
     DEFINE_PROP_BOOL("migratable", VHostSCSICommon, migratable, false),
+    DEFINE_PROP_INT32("virtqueue_workers", VirtIOSCSICommon,
+                      conf.virtqueue_workers, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c
index 31b33bde37b2..0dc9acfca7ec 100644
--- a/hw/virtio/vhost-backend.c
+++ b/hw/virtio/vhost-backend.c
@@ -150,6 +150,13 @@ static int vhost_kernel_set_vring_busyloop_timeout(struct vhost_dev *dev,
     return vhost_kernel_call(dev, VHOST_SET_VRING_BUSYLOOP_TIMEOUT, s);
 }
 
+static int vhost_kernel_set_vring_worker(struct vhost_dev *dev,
+                                         struct vhost_vring_worker *worker)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_WORKER, worker);
+}
+
+
 static int vhost_kernel_set_features(struct vhost_dev *dev,
                                      uint64_t features)
 {
@@ -311,6 +318,7 @@ static const VhostOps kernel_ops = {
         .vhost_set_vring_call = vhost_kernel_set_vring_call,
         .vhost_set_vring_busyloop_timeout =
                                 vhost_kernel_set_vring_busyloop_timeout,
+        .vhost_set_vring_worker = vhost_kernel_set_vring_worker,
         .vhost_set_features = vhost_kernel_set_features,
         .vhost_get_features = vhost_kernel_get_features,
         .vhost_set_backend_cap = vhost_kernel_set_backend_cap,
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index 8a6f8e2a7a30..375fd6e79d8f 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -33,6 +33,7 @@ struct vhost_memory;
 struct vhost_vring_file;
 struct vhost_vring_state;
 struct vhost_vring_addr;
+struct vhost_vring_worker;
 struct vhost_scsi_target;
 struct vhost_iotlb_msg;
 struct vhost_virtqueue;
@@ -70,6 +71,8 @@ typedef int (*vhost_set_vring_call_op)(struct vhost_dev *dev,
                                        struct vhost_vring_file *file);
 typedef int (*vhost_set_vring_busyloop_timeout_op)(struct vhost_dev *dev,
                                                    struct vhost_vring_state *r);
+typedef int (*vhost_set_vring_worker_op)(struct vhost_dev *dev,
+                                         struct vhost_vring_worker *worker);
 typedef int (*vhost_set_features_op)(struct vhost_dev *dev,
                                      uint64_t features);
 typedef int (*vhost_get_features_op)(struct vhost_dev *dev,
@@ -145,6 +148,7 @@ typedef struct VhostOps {
     vhost_set_vring_kick_op vhost_set_vring_kick;
     vhost_set_vring_call_op vhost_set_vring_call;
     vhost_set_vring_busyloop_timeout_op vhost_set_vring_busyloop_timeout;
+    vhost_set_vring_worker_op vhost_set_vring_worker;
     vhost_set_features_op vhost_set_features;
     vhost_get_features_op vhost_get_features;
     vhost_set_backend_cap_op vhost_set_backend_cap;
diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h
index 543681bc1838..694221601dad 100644
--- a/include/hw/virtio/virtio-scsi.h
+++ b/include/hw/virtio/virtio-scsi.h
@@ -58,6 +58,7 @@ struct VirtIOSCSIConf {
 #ifdef CONFIG_VHOST_SCSI
     char *vhostfd;
     char *wwpn;
+    int virtqueue_workers;
 #endif
     CharBackend chardev;
     uint32_t boot_tpgt;
diff --git a/include/standard-headers/linux/vhost_types.h b/include/standard-headers/linux/vhost_types.h
index 0bd2684a2ae4..0d81ff6b2f1f 100644
--- a/include/standard-headers/linux/vhost_types.h
+++ b/include/standard-headers/linux/vhost_types.h
@@ -47,6 +47,15 @@ struct vhost_vring_addr {
 	uint64_t log_guest_addr;
 };
 
+struct vhost_vring_worker {
+	unsigned int index;
+	/*
+	 * The pid of the vhost worker that the vq will be bound to. If -1,
+	 * a new worker will be created and it's pid will be returned in pid.
+	 */
+	pid_t pid;
+};
+
 /* no alignment requirement */
 struct vhost_iotlb_msg {
 	uint64_t iova;
diff --git a/linux-headers/linux/vhost.h b/linux-headers/linux/vhost.h
index c998860d7bbc..24569f89611b 100644
--- a/linux-headers/linux/vhost.h
+++ b/linux-headers/linux/vhost.h
@@ -70,6 +70,13 @@
 #define VHOST_VRING_BIG_ENDIAN 1
 #define VHOST_SET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x13, struct vhost_vring_state)
 #define VHOST_GET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x14, struct vhost_vring_state)
+/* Create/bind a vhost worker to a virtqueue. If pid > 0 and matches an existing
+ * vhost_worker thread it will be bound to the vq. If pid is -1, then a new
+ * worker will be created and bound to the vq.
+ */
+#define VHOST_SET_VRING_WORKER _IOWR(VHOST_VIRTIO, 0x15, struct vhost_vring_worker)
+/* Return the vqs worker's pid. If no worker is set pid is -1 */
+#define VHOST_GET_VRING_WORKER _IOR(VHOST_VIRTIO, 0x16, struct vhost_vring_worker)
 
 /* The following ioctls use eventfd file descriptors to signal and poll
  * for events. */
-- 
2.25.1

_______________________________________________
Virtualization mailing list
Virtualization@xxxxxxxxxxxxxxxxxxxxxxxxxx
https://lists.linuxfoundation.org/mailman/listinfo/virtualization



[Index of Archives]     [KVM Development]     [Libvirt Development]     [Libvirt Users]     [CentOS Virtualization]     [Netdev]     [Ethernet Bridging]     [Linux Wireless]     [Kernel Newbies]     [Security]     [Linux for Hams]     [Netfilter]     [Bugtraq]     [Yosemite Forum]     [MIPS Linux]     [ARM Linux]     [Linux RAID]     [Linux Admin]     [Samba]

  Powered by Linux