[PATCH v2 2/2] add pci passthrough impl to libxl

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Add pci passthrough implementation to libxl driver

Changes to v1:
* Add device-attach/detach a pci hostdev implementataion to libxl driver
* <driver name='xen' /> should be specified in <hostdev> definition to keep
  consistency with qemu changes (use <driver name='kvm/vfio' /> to decide stub
  driver pci-stub/vfio). According to name='xen', it will use 'pciback'. 


Signed-off-by: Chunyan Liu <cyliu@xxxxxxxx>
---
 src/libxl/libxl_conf.c   |  106 +++++++++++++++++++
 src/libxl/libxl_conf.h   |    3 +-
 src/libxl/libxl_driver.c |  250 +++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 355 insertions(+), 4 deletions(-)

diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
index 1a7e430..4f2fc91 100644
--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
@@ -708,6 +708,108 @@ error:
     return -1;
 }
 
+int libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev)
+{
+    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
+        return -1;
+    if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+        return -1;
+
+    pcidev->domain = hostdev->source.subsys.u.pci.addr.domain;
+    pcidev->bus = hostdev->source.subsys.u.pci.addr.bus;
+    pcidev->dev = hostdev->source.subsys.u.pci.addr.slot;
+    pcidev->func = hostdev->source.subsys.u.pci.addr.function;
+
+    return 0;
+}
+
+static int
+libxlMakePciList(virDomainDefPtr def, libxl_domain_config *d_config)
+{
+    virDomainHostdevDefPtr *l_hostdevs = def->hostdevs;
+    size_t nhostdevs = def->nhostdevs;
+    size_t npcidevs = 0;
+    libxl_device_pci *x_pcidevs;
+    int i, j;
+
+    if (nhostdevs == 0)
+        return 0;
+
+    if (VIR_ALLOC_N(x_pcidevs, nhostdevs) < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    for (i = 0, j = 0; i < nhostdevs; i++) {
+        if (l_hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
+            continue;
+        if (l_hostdevs[i]->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+            continue;
+
+        libxl_device_pci_init(&x_pcidevs[j]);
+
+        if (libxlMakePci(l_hostdevs[i], &x_pcidevs[j]) < 0)
+            goto error;
+
+        npcidevs++;
+        j++;
+    }
+
+    VIR_SHRINK_N(x_pcidevs, nhostdevs, nhostdevs - npcidevs);
+    d_config->pcidevs = x_pcidevs;
+    d_config->num_pcidevs = npcidevs;
+
+    return 0;
+
+error:
+    for (i = 0; i < npcidevs; i++) {
+        libxl_device_pci_dispose(&x_pcidevs[i]);
+    }
+    VIR_FREE(x_pcidevs);
+    return -1;
+}
+
+/*
+static int
+libxlMakePciList(virDomainDefPtr def, libxl_domain_config *d_config)
+{
+    virDomainHostdevDefPtr *l_hostdevs = def->hostdevs;
+    int nhostdevs = def->nhostdevs;
+    libxl_device_pci *x_pcidevs;
+    int i;
+
+    if (nhostdevs == 0)
+        return 0;
+
+    if (VIR_ALLOC_N(x_pcidevs, nhostdevs) < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    d_config->num_pcidevs = 0;
+    d_config->pcidevs = x_pcidevs;
+
+    for (i = 0 ; i < nhostdevs ; i++) {
+        virDomainHostdevDefPtr hostdev = l_hostdevs[i];
+        libxl_device_pci pcidev;
+
+        if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
+            continue;
+        if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+            continue;
+
+        pcidev.domain = hostdev->source.subsys.u.pci.addr.domain;
+        pcidev.bus = hostdev->source.subsys.u.pci.addr.bus;
+        pcidev.dev = hostdev->source.subsys.u.pci.addr.slot;
+        pcidev.func = hostdev->source.subsys.u.pci.addr.function;
+
+        d_config->pcidevs[d_config->num_pcidevs] = pcidev;
+        d_config->num_pcidevs ++;
+    }
+
+    return 0;
+}
+*/
 virCapsPtr
 libxlMakeCapabilities(libxl_ctx *ctx)
 {
@@ -766,6 +868,10 @@ libxlBuildDomainConfig(libxlDriverPrivatePtr driver,
         return -1;
     }
 
+    if (libxlMakePciList(def, d_config) < 0) {
+        return -1;
+    }
+
     d_config->on_reboot = def->onReboot;
     d_config->on_poweroff = def->onPoweroff;
     d_config->on_crash = def->onCrash;
diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h
index fed878d..7d7a021 100644
--- a/src/libxl/libxl_conf.h
+++ b/src/libxl/libxl_conf.h
@@ -119,7 +119,8 @@ libxlMakeNic(virDomainNetDefPtr l_nic, libxl_device_nic *x_nic);
 int
 libxlMakeVfb(libxlDriverPrivatePtr driver,
              virDomainGraphicsDefPtr l_vfb, libxl_device_vfb *x_vfb);
-
+int
+libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev);
 int
 libxlBuildDomainConfig(libxlDriverPrivatePtr driver,
                        virDomainDefPtr def, libxl_domain_config *d_config);
diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
index b39e005..a94b214 100644
--- a/src/libxl/libxl_driver.c
+++ b/src/libxl/libxl_driver.c
@@ -45,6 +45,7 @@
 #include "virtypedparam.h"
 #include "viruri.h"
 #include "virstring.h"
+#include "virhostdevmanager.h"
 
 #define VIR_FROM_THIS VIR_FROM_LIBXL
 
@@ -685,13 +686,17 @@ libxlVmReap(libxlDriverPrivatePtr driver,
             virDomainShutoffReason reason)
 {
     libxlDomainObjPrivatePtr priv = vm->privateData;
-
+    virHostdevManagerPtr hostdev_mgr;
+ 
     if (libxl_domain_destroy(priv->ctx, vm->def->id, NULL) < 0) {
         virReportError(VIR_ERR_INTERNAL_ERROR,
                        _("Unable to cleanup domain %d"), vm->def->id);
         return -1;
     }
 
+    hostdev_mgr = virHostdevManagerGetDefault();
+    virHostdevManagerReAttachHostdevs(hostdev_mgr, "xenlight", vm->def, VIR_SP_PCI_HOSTDEV);
+
     libxlVmCleanup(driver, vm, reason);
     return 0;
 }
@@ -902,6 +907,7 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
     char *managed_save_path = NULL;
     int managed_save_fd = -1;
     libxlDomainObjPrivatePtr priv = vm->privateData;
+    virHostdevManagerPtr hostdev_mgr;
 
     /* If there is a managed saved state restore it instead of starting
      * from scratch. The old state is removed once the restoring succeeded. */
@@ -956,6 +962,12 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
         goto error;
     }
 
+    VIR_DEBUG("Preparing host PCI devices");
+    hostdev_mgr = virHostdevManagerGetDefault();
+    if (virHostdevManagerPrepareHostdevs(hostdev_mgr, "xenlight", vm->def, VIR_SP_PCI_HOSTDEV ) < 0)
+        goto error;
+
+
     /* use as synchronous operations => ao_how = NULL and no intermediate reports => ao_progress = NULL */
 
     if (restore_fd < 0)
@@ -1049,6 +1061,7 @@ libxlReconnectDomain(virDomainObjPtr vm,
     libxl_dominfo d_info;
     int len;
     uint8_t *data = NULL;
+    virHostdevManagerPtr hostdev_mgr;
 
     virObjectLock(vm);
 
@@ -1071,6 +1084,12 @@ libxlReconnectDomain(virDomainObjPtr vm,
 
     /* Update domid in case it changed (e.g. reboot) while we were gone? */
     vm->def->id = d_info.domid;
+
+    /* Update hostdev state */
+    hostdev_mgr = virHostdevManagerGetDefault();
+    if (virHostdevManagerPrepareHostdevs(hostdev_mgr, "xenlight", vm->def, VIR_SP_PCI_HOSTDEV ) < 0)
+        goto out;
+
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNKNOWN);
 
     if (!driver->nactive && driver->inhibitCallback)
@@ -3307,6 +3326,114 @@ cleanup:
 }
 
 static int
+libxlDomainAttachHostPciDevice(libxlDomainObjPrivatePtr priv,
+                               virDomainObjPtr vm,
+                               virDomainHostdevDefPtr hostdev)
+{
+    libxl_device_pci pcidev;
+    virDomainHostdevDefPtr found;
+    virHostdevManagerPtr hostdev_mgr;
+
+    if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+        return -1;
+
+    if (virDomainHostdevFind(vm->def, hostdev, &found) >= 0) {
+        virReportError(VIR_ERR_OPERATION_FAILED,
+                       _("target pci device %.4x:%.2x:%.2x.%.1x already exists"),
+                       hostdev->source.subsys.u.pci.addr.domain, 
+                       hostdev->source.subsys.u.pci.addr.bus,
+                       hostdev->source.subsys.u.pci.addr.slot,
+                       hostdev->source.subsys.u.pci.addr.function);
+        return -1;
+    }
+
+    if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0) {
+        virReportOOMError();
+        return -1;
+    }
+
+    hostdev_mgr = virHostdevManagerGetDefault();
+    if (virHostdevManagerPreparePciHostdevs(hostdev_mgr, "xenlight",
+                                            vm->def->name, vm->def->uuid,
+                                            &hostdev, 1, 0) < 0) {
+        virReportError(VIR_ERR_OPERATION_FAILED,
+                       "Failed to prepare hostdev");
+        goto cleanup;
+    }
+
+    if (libxlMakePci(hostdev, &pcidev) < 0)
+        goto reattach_hostdev;
+
+    if (libxl_device_pci_add(priv->ctx, vm->def->id, &pcidev, 0) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("libxenlight failed to attach pci device %.4x:%.2x:%.2x.%.1x"),
+                       hostdev->source.subsys.u.pci.addr.domain,
+                       hostdev->source.subsys.u.pci.addr.bus,
+                       hostdev->source.subsys.u.pci.addr.slot, 
+                       hostdev->source.subsys.u.pci.addr.function);
+        goto reattach_hostdev;
+    }
+    
+    vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
+    return 0;
+
+reattach_hostdev:
+    virHostdevManagerReAttachPciHostdevs(hostdev_mgr, "xenlight", 
+                                         vm->def->name, &hostdev, 1);
+
+cleanup:
+    return -1;
+}
+
+static int 
+libxlDomainAttachHostDevice(libxlDomainObjPrivatePtr priv,
+                            virDomainObjPtr vm,
+                            virDomainDeviceDefPtr dev)
+{
+    virDomainHostdevDefPtr hostdev = dev->data.hostdev;
+
+    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("hostdev mode '%s' not supported"),
+                       virDomainHostdevModeTypeToString(hostdev->mode));
+        return -1;
+    }
+/*
+    if (qemuSetupHostdevCGroup(vm, hostdev) < 0)
+        return -1;
+
+    if (virSecurityManagerSetHostdevLabel(driver->securityManager,
+                                          vm->def, hostdev, NULL) < 0)
+        goto cleanup;
+*/
+    switch (hostdev->source.subsys.type) {
+    case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
+        if (libxlDomainAttachHostPciDevice(priv, vm, hostdev) < 0)
+            goto error;
+        break;
+
+    default:
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("hostdev subsys type '%s' not supported"),
+                       virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type));
+        goto error;
+    }
+
+    return 0;
+
+error:
+/*    if (virSecurityManagerRestoreHostdevLabel(driver->securityManager,
+                                              vm->def, hostdev, NULL) < 0)
+        VIR_WARN("Unable to restore host device labelling on hotplug fail");
+
+cleanup:
+    if (qemuTeardownHostdevCgroup(vm, hostdev) < 0)
+        VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
+*/
+    return -1;
+}
+
+static int
 libxlDomainDetachDeviceDiskLive(libxlDomainObjPrivatePtr priv,
                                 virDomainObjPtr vm, virDomainDeviceDefPtr dev)
 {
@@ -3372,7 +3499,11 @@ libxlDomainAttachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
             if (!ret)
                 dev->data.disk = NULL;
             break;
-
+        case VIR_DOMAIN_DEVICE_HOSTDEV:
+            ret = libxlDomainAttachHostDevice(priv, vm, dev);
+            if (!ret)
+                dev->data.hostdev = NULL;
+            break;
         default:
             virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                            _("device type '%s' cannot be attached"),
@@ -3387,6 +3518,8 @@ static int
 libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
 {
     virDomainDiskDefPtr disk;
+    virDomainHostdevDefPtr hostdev;
+    virDomainHostdevDefPtr found;
 
     switch (dev->type) {
         case VIR_DOMAIN_DEVICE_DISK:
@@ -3403,6 +3536,25 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
             /* vmdef has the pointer. Generic codes for vmdef will do all jobs */
             dev->data.disk = NULL;
             break;
+        case VIR_DOMAIN_DEVICE_HOSTDEV:
+            hostdev = dev->data.hostdev;
+
+            if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+                return -1;
+
+            if (virDomainHostdevFind(vmdef, hostdev, &found) >= 0) {
+                virReportError(VIR_ERR_OPERATION_FAILED,
+                               _("target pci device %.4x:%.2x:%.2x.%.1x\
+                                  already exists"),
+                               hostdev->source.subsys.u.pci.addr.domain,
+                               hostdev->source.subsys.u.pci.addr.bus,
+                               hostdev->source.subsys.u.pci.addr.slot,
+                               hostdev->source.subsys.u.pci.addr.function);
+                return -1;
+            }
+
+            virDomainHostdevInsert(vmdef, hostdev);
+            break;
 
         default:
             virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
@@ -3413,6 +3565,95 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
 }
 
 static int
+libxlDomainDetachHostPciDevice(libxlDomainObjPrivatePtr priv,
+                               virDomainObjPtr vm,
+                               virDomainHostdevDefPtr hostdev)
+{
+    virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys;
+    libxl_device_pci pcidev;
+    virDomainHostdevDefPtr detach;
+    int idx;
+    virHostdevManagerPtr hostdev_mgr;
+
+/*    if (qemuIsMultiFunctionDevice(vm->def, detach->info)) {
+        virReportError(VIR_ERR_OPERATION_FAILED,
+                       _("cannot hot unplug multifunction PCI device: %.4x:%.2x:%.2x.%.1x"),
+                       subsys->u.pci.addr.domain, subsys->u.pci.addr.bus,
+                       subsys->u.pci.addr.slot, subsys->u.pci.addr.function);
+        goto cleanup;
+    }
+*/
+    if (subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
+        return -1;
+
+    idx = virDomainHostdevFind(vm->def, hostdev, &detach);
+    if (idx < 0) {
+        virReportError(VIR_ERR_OPERATION_FAILED,
+                       _("host pci device %.4x:%.2x:%.2x.%.1x not found"),
+                       subsys->u.pci.addr.domain, subsys->u.pci.addr.bus,
+                       subsys->u.pci.addr.slot, subsys->u.pci.addr.function);
+        return -1;
+    }
+    
+   if (libxlMakePci(detach, &pcidev) < 0)
+       goto cleanup;
+
+   if (libxl_device_pci_remove(priv->ctx, vm->def->id, &pcidev, 0) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("libxenlight failed to detach pci device\
+                          %.4x:%.2x:%.2x.%.1x"),
+                       subsys->u.pci.addr.domain, subsys->u.pci.addr.bus,
+                       subsys->u.pci.addr.slot, subsys->u.pci.addr.function);
+        goto cleanup;
+    }
+
+    virDomainHostdevRemove(vm->def, idx);
+
+    hostdev_mgr = virHostdevManagerGetDefault();
+    virHostdevManagerReAttachPciHostdevs(hostdev_mgr, "xenlight",
+                                         vm->def->name, &hostdev, 1);
+
+/*
+    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE) &&
+        qemuDomainPCIAddressReleaseSlot(priv->pciaddrs,
+                                        &detach->info->addr.pci) < 0)
+        VIR_WARN("Unable to release PCI address on host device");
+*/
+
+    return 0;
+
+cleanup:
+    virDomainHostdevDefFree(detach);
+    return -1;
+}
+
+static int libxlDomainDetachHostDevice(libxlDomainObjPrivatePtr priv,
+                                       virDomainObjPtr vm,
+                                       virDomainDeviceDefPtr dev)
+{
+    virDomainHostdevDefPtr hostdev = dev->data.hostdev;
+    virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys;
+
+    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
+        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                       _("hostdev mode '%s' not supported"),
+                       virDomainHostdevModeTypeToString(hostdev->mode));
+        return -1;
+    }
+
+    switch (subsys->type) {
+        case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
+            return libxlDomainDetachHostPciDevice(priv, vm, hostdev);
+        default:
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+                           _("unexpected hostdev type %d"), subsys->type);
+            break;
+    }
+
+    return -1;
+}
+
+static int
 libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
                             virDomainDeviceDefPtr dev)
 {
@@ -3422,7 +3663,9 @@ libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
         case VIR_DOMAIN_DEVICE_DISK:
             ret = libxlDomainDetachDeviceDiskLive(priv, vm, dev);
             break;
-
+        case VIR_DOMAIN_DEVICE_HOSTDEV:
+            ret = libxlDomainDetachHostDevice(priv, vm, dev);
+            break;
         default:
             virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                            _("device type '%s' cannot be detached"),
@@ -3433,6 +3676,7 @@ libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
     return ret;
 }
 
+
 static int
 libxlDomainDetachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
 {
-- 
1.6.0.2

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list




[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]