This patch adds support for the proposed ioctl that allows userspace
to create virtqueue workers. For vhost-scsi you can set virtqueue_workers
to:
0: default behavior where we have 1 worker for all vqs.
-1: create a worker per vq
>0: create N workers and allow the vqs to share them by assigning a
vq a worker by just doing round robin.
TODO:
- Allow sharing workers across devices.
- Bind to specific CPUs. Commands like "virsh emulatorpin" allow us
to set the group of vhost threads to different CPUs. But we can also
set a specific vq's worker to run on a CPU.
- I'm handling old kernel by just checking for EPERM. Does this require
a feature?
Signed-off-by: Mike Christie <michael.christie@xxxxxxxxxx>
---
hw/scsi/vhost-scsi.c | 85 +++++++++++++++++++-
hw/virtio/vhost-backend.c | 8 ++
include/hw/virtio/vhost-backend.h | 4 +
include/hw/virtio/virtio-scsi.h | 1 +
include/standard-headers/linux/vhost_types.h | 9 +++
linux-headers/linux/vhost.h | 7 ++
6 files changed, 111 insertions(+), 3 deletions(-)
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 4d70fa036bbe..9e3653d158c3 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -163,6 +163,76 @@ static const VMStateDescription vmstate_virtio_vhost_scsi = {
.pre_save = vhost_scsi_pre_save,
};
+static int vhost_scsi_set_workers(VHostSCSICommon *vsc, int vq_workers)
+{
+ struct vhost_dev *dev = &vsc->dev;
+ int worker_index = 0, num_workers = 0;
+ struct vhost_vring_worker w;
+ pid_t *workers = NULL;
+ int i, ret;
+
+ if (vq_workers < -1)
+ return -EINVAL;
+
+ if (vq_workers > 0) {
+ if (vq_workers > dev->nvqs)
+ vq_workers = dev->nvqs;
+
+ workers = g_malloc0(vq_workers * sizeof(pid_t));
+ }
+
+ w.pid = -1;
+ for (i = 0; i < dev->nvqs; i++) {
+ w.index = i;
+
+ switch (vq_workers) {
+ case -1:
+ /*
+ * ctl/evt share the first worker since it will be rare for them
+ * to send cmds while IO is running. The rest of the vqs get their
+ * own worker.
+ */
+ if (i > VHOST_SCSI_VQ_NUM_FIXED)
+ w.pid = -1;
+ break;
+ case 0:
+ /* All vqs share 1 worker. Pass back the pid we got the first run */
+ break;
+ default:
+ /* Each worker handles N vqs. */
+ if (num_workers == vq_workers) {
+ w.pid = workers[worker_index];
+
+ worker_index++;
+ if (worker_index == vq_workers)
+ worker_index = 0;
+ } else {
+ w.pid = -1;
+ }
+
+ break;
+ }
+
+ ret = dev->vhost_ops->vhost_set_vring_worker(dev, &w);
+ /* Ignore for now. Add feature in final patch */
+ if (ret == -EPERM) {
+ ret = 0;
+ goto free_workers;
+ } else if (ret)
+ goto free_workers;
+
+ if (vq_workers > 0 && num_workers < vq_workers) {
+ workers[num_workers] = w.pid;
+ num_workers++;
+ }
+ }
+
+free_workers:
+ if (workers)
+ g_free(workers);
+ return ret;
+}
+
static void vhost_scsi_realize(DeviceState *dev, Error **errp)
{
VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(dev);
@@ -226,6 +296,13 @@ static void vhost_scsi_realize(DeviceState *dev, Error **errp)
goto free_vqs;
}
+ ret = vhost_scsi_set_workers(vsc, vs->conf.virtqueue_workers);
+ if (ret < 0) {
+ error_setg(errp, "vhost-scsi: vhost worker setup failed: %s",
+ strerror(-ret));
+ goto free_vqs;
+ }
+
/* At present, channel and lun both are 0 for bootable vhost-scsi disk */
vsc->channel = 0;
vsc->lun = 0;
@@ -271,18 +348,20 @@ static Property vhost_scsi_properties[] = {
DEFINE_PROP_STRING("wwpn", VirtIOSCSICommon, conf.wwpn),
DEFINE_PROP_UINT32("boot_tpgt", VirtIOSCSICommon, conf.boot_tpgt, 0),
DEFINE_PROP_UINT32("num_queues", VirtIOSCSICommon, conf.num_queues,
- VIRTIO_SCSI_AUTO_NUM_QUEUES),
+ 8),
DEFINE_PROP_UINT32("virtqueue_size", VirtIOSCSICommon, conf.virtqueue_size,
- 128),
+ 1024),
DEFINE_PROP_BOOL("seg_max_adjust", VirtIOSCSICommon, conf.seg_max_adjust,
true),
DEFINE_PROP_UINT32("max_sectors", VirtIOSCSICommon, conf.max_sectors,
0xFFFF),
- DEFINE_PROP_UINT32("cmd_per_lun", VirtIOSCSICommon, conf.cmd_per_lun, 128),
+ DEFINE_PROP_UINT32("cmd_per_lun", VirtIOSCSICommon, conf.cmd_per_lun, 1024),
DEFINE_PROP_BIT64("t10_pi", VHostSCSICommon, host_features,
VIRTIO_SCSI_F_T10_PI,
false),
DEFINE_PROP_BOOL("migratable", VHostSCSICommon, migratable, false),
+ DEFINE_PROP_INT32("virtqueue_workers", VirtIOSCSICommon,
+ conf.virtqueue_workers, 0),
DEFINE_PROP_END_OF_LIST(),
};
diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c
index 31b33bde37b2..0dc9acfca7ec 100644
--- a/hw/virtio/vhost-backend.c
+++ b/hw/virtio/vhost-backend.c
@@ -150,6 +150,13 @@ static int vhost_kernel_set_vring_busyloop_timeout(struct vhost_dev *dev,
return vhost_kernel_call(dev, VHOST_SET_VRING_BUSYLOOP_TIMEOUT, s);
}
+static int vhost_kernel_set_vring_worker(struct vhost_dev *dev,
+ struct vhost_vring_worker *worker)
+{
+ return vhost_kernel_call(dev, VHOST_SET_VRING_WORKER, worker);
+}
+
+
static int vhost_kernel_set_features(struct vhost_dev *dev,
uint64_t features)
{
@@ -311,6 +318,7 @@ static const VhostOps kernel_ops = {
.vhost_set_vring_call = vhost_kernel_set_vring_call,
.vhost_set_vring_busyloop_timeout =
vhost_kernel_set_vring_busyloop_timeout,
+ .vhost_set_vring_worker = vhost_kernel_set_vring_worker,
.vhost_set_features = vhost_kernel_set_features,
.vhost_get_features = vhost_kernel_get_features,
.vhost_set_backend_cap = vhost_kernel_set_backend_cap,
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index 8a6f8e2a7a30..375fd6e79d8f 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -33,6 +33,7 @@ struct vhost_memory;
struct vhost_vring_file;
struct vhost_vring_state;
struct vhost_vring_addr;
+struct vhost_vring_worker;
struct vhost_scsi_target;
struct vhost_iotlb_msg;
struct vhost_virtqueue;
@@ -70,6 +71,8 @@ typedef int (*vhost_set_vring_call_op)(struct vhost_dev *dev,
struct vhost_vring_file *file);
typedef int (*vhost_set_vring_busyloop_timeout_op)(struct vhost_dev *dev,
struct vhost_vring_state *r);
+typedef int (*vhost_set_vring_worker_op)(struct vhost_dev *dev,
+ struct vhost_vring_worker *worker);
typedef int (*vhost_set_features_op)(struct vhost_dev *dev,
uint64_t features);
typedef int (*vhost_get_features_op)(struct vhost_dev *dev,
@@ -145,6 +148,7 @@ typedef struct VhostOps {
vhost_set_vring_kick_op vhost_set_vring_kick;
vhost_set_vring_call_op vhost_set_vring_call;
vhost_set_vring_busyloop_timeout_op vhost_set_vring_busyloop_timeout;
+ vhost_set_vring_worker_op vhost_set_vring_worker;
vhost_set_features_op vhost_set_features;
vhost_get_features_op vhost_get_features;
vhost_set_backend_cap_op vhost_set_backend_cap;
diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h
index 543681bc1838..694221601dad 100644
--- a/include/hw/virtio/virtio-scsi.h
+++ b/include/hw/virtio/virtio-scsi.h
@@ -58,6 +58,7 @@ struct VirtIOSCSIConf {
#ifdef CONFIG_VHOST_SCSI
char *vhostfd;
char *wwpn;
+ int virtqueue_workers;
#endif
CharBackend chardev;
uint32_t boot_tpgt;
diff --git a/include/standard-headers/linux/vhost_types.h b/include/standard-headers/linux/vhost_types.h
index 0bd2684a2ae4..0d81ff6b2f1f 100644
--- a/include/standard-headers/linux/vhost_types.h
+++ b/include/standard-headers/linux/vhost_types.h
@@ -47,6 +47,15 @@ struct vhost_vring_addr {
uint64_t log_guest_addr;
};
+struct vhost_vring_worker {
+ unsigned int index;
+ /*
+ * The pid of the vhost worker that the vq will be bound to. If -1,
+ * a new worker will be created and it's pid will be returned in pid.
+ */
+ pid_t pid;
+};
+
/* no alignment requirement */
struct vhost_iotlb_msg {
uint64_t iova;
diff --git a/linux-headers/linux/vhost.h b/linux-headers/linux/vhost.h
index c998860d7bbc..24569f89611b 100644
--- a/linux-headers/linux/vhost.h
+++ b/linux-headers/linux/vhost.h
@@ -70,6 +70,13 @@
#define VHOST_VRING_BIG_ENDIAN 1
#define VHOST_SET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x13, struct vhost_vring_state)
#define VHOST_GET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x14, struct vhost_vring_state)
+/* Create/bind a vhost worker to a virtqueue. If pid > 0 and matches an existing
+ * vhost_worker thread it will be bound to the vq. If pid is -1, then a new
+ * worker will be created and bound to the vq.
+ */
+#define VHOST_SET_VRING_WORKER _IOWR(VHOST_VIRTIO, 0x15, struct vhost_vring_worker)
+/* Return the vqs worker's pid. If no worker is set pid is -1 */
+#define VHOST_GET_VRING_WORKER _IOR(VHOST_VIRTIO, 0x16, struct vhost_vring_worker)
/* The following ioctls use eventfd file descriptors to signal and poll
* for events. */
--
2.25.1
_______________________________________________
Virtualization mailing list
Virtualization@xxxxxxxxxxxxxxxxxxxxxxxxxx
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
