Re: [PATCHv4 2/4] add pci passthrough to libxl driver

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



cyliu@xxxxxxxx wrote:
> From: Chunyan Liu <cyliu@xxxxxxxx>
>
> Add pci passthrough to libxl driver, support attach-device, detach-device and
> start a vm with pci hostdev specified.
>   

While testing this patch, I noticed it only works with "managed"
hostdevs, i.e. hostdevs with managed='yes' attribute.  Should this patch
include an implementation of the nodeDevice* functions so users can e.g.
do 'virsh nodedev-detach' prior to attaching or starting a domain with
an "unmanaged" hostdev?  Alternatively, the nodeDevice* functions could
be implemented in a patch that precedes this one.

Regards,
Jim

> Signed-off-by: Chunyan Liu <cyliu@xxxxxxxx>
> ---
>  po/POTFILES.in           |    2 +-
>  src/libxl/libxl_conf.c   |   63 +++++++++++
>  src/libxl/libxl_conf.h   |    4 +
>  src/libxl/libxl_driver.c |  262 +++++++++++++++++++++++++++++++++++++++++++++-
>  4 files changed, 329 insertions(+), 2 deletions(-)
>
> diff --git a/po/POTFILES.in b/po/POTFILES.in
> index 6799d47..50030f8 100644
> --- a/po/POTFILES.in
> +++ b/po/POTFILES.in
> @@ -157,6 +157,7 @@ src/util/vireventpoll.c
>  src/util/virfile.c
>  src/util/virhash.c
>  src/util/virhook.c
> +src/util/virhostdev.c
>  src/util/viridentity.c
>  src/util/virinitctl.c
>  src/util/viriptables.c
> @@ -194,7 +195,6 @@ src/util/viruri.c
>  src/util/virusb.c
>  src/util/virutil.c
>  src/util/virxml.c
> -src/util/virhostdev.c
>  src/vbox/vbox_MSCOMGlue.c
>  src/vbox/vbox_XPCOMCGlue.c
>  src/vbox/vbox_driver.c
> diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
> index 73e2d49..f76f432 100644
> --- a/src/libxl/libxl_conf.c
> +++ b/src/libxl/libxl_conf.c
> @@ -869,6 +869,66 @@ error:
>      return -1;
>  }
>  
> +int
> +libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev)
> +{
> +    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
> +        return -1;
> +    if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
> +        return -1;
> +
> +    pcidev->domain = hostdev->source.subsys.u.pci.addr.domain;
> +    pcidev->bus = hostdev->source.subsys.u.pci.addr.bus;
> +    pcidev->dev = hostdev->source.subsys.u.pci.addr.slot;
> +    pcidev->func = hostdev->source.subsys.u.pci.addr.function;
> +
> +    return 0;
> +}
> +
> +static int
> +libxlMakePciList(virDomainDefPtr def, libxl_domain_config *d_config)
> +{
> +    virDomainHostdevDefPtr *l_hostdevs = def->hostdevs;
> +    size_t nhostdevs = def->nhostdevs;
> +    size_t npcidevs = 0;
> +    libxl_device_pci *x_pcidevs;
> +    size_t i, j;
> +
> +    if (nhostdevs == 0)
> +        return 0;
> +
> +    if (VIR_ALLOC_N(x_pcidevs, nhostdevs) < 0)
> +        return -1;
> +
> +    for (i = 0, j = 0; i < nhostdevs; i++) {
> +        if (l_hostdevs[i]->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
> +            continue;
> +        if (l_hostdevs[i]->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
> +            continue;
> +
> +        libxl_device_pci_init(&x_pcidevs[j]);
> +
> +        if (libxlMakePci(l_hostdevs[i], &x_pcidevs[j]) < 0)
> +            goto error;
> +
> +        npcidevs++;
> +        j++;
> +    }
> +
> +    VIR_SHRINK_N(x_pcidevs, nhostdevs, nhostdevs - npcidevs);
> +    d_config->pcidevs = x_pcidevs;
> +    d_config->num_pcidevs = npcidevs;
> +
> +    return 0;
> +
> +error:
> +    for (i = 0; i < npcidevs; i++)
> +        libxl_device_pci_dispose(&x_pcidevs[i]);
> +
> +    VIR_FREE(x_pcidevs);
> +    return -1;
> +}
> +
>  virCapsPtr
>  libxlMakeCapabilities(libxl_ctx *ctx)
>  {
> @@ -913,6 +973,9 @@ libxlBuildDomainConfig(libxlDriverPrivatePtr driver,
>      if (libxlMakeVfbList(driver, def, d_config) < 0)
>          return -1;
>  
> +    if (libxlMakePciList(def, d_config) < 0)
> +        return -1;
> +
>      d_config->on_reboot = def->onReboot;
>      d_config->on_poweroff = def->onPoweroff;
>      d_config->on_crash = def->onCrash;
> diff --git a/src/libxl/libxl_conf.h b/src/libxl/libxl_conf.h
> index 9d72b72..39d09fe 100644
> --- a/src/libxl/libxl_conf.h
> +++ b/src/libxl/libxl_conf.h
> @@ -37,6 +37,7 @@
>  # include "virchrdev.h"
>  
>  
> +# define LIBXL_DRIVER_NAME "xenlight"
>  # define LIBXL_VNC_PORT_MIN  5900
>  # define LIBXL_VNC_PORT_MAX  65535
>  
> @@ -132,6 +133,9 @@ libxlMakeVfb(libxlDriverPrivatePtr driver,
>               virDomainGraphicsDefPtr l_vfb, libxl_device_vfb *x_vfb);
>  
>  int
> +libxlMakePci(virDomainHostdevDefPtr hostdev, libxl_device_pci *pcidev);
> +
> +int
>  libxlBuildDomainConfig(libxlDriverPrivatePtr driver,
>                         virDomainObjPtr vm, libxl_domain_config *d_config);
>  
> diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c
> index f09ad61..029fa0c 100644
> --- a/src/libxl/libxl_driver.c
> +++ b/src/libxl/libxl_driver.c
> @@ -49,6 +49,7 @@
>  #include "virstring.h"
>  #include "virsysinfo.h"
>  #include "viraccessapicheck.h"
> +#include "virhostdev.h"
>  
>  #define VIR_FROM_THIS VIR_FROM_LIBXL
>  
> @@ -742,6 +743,7 @@ libxlVmReap(libxlDriverPrivatePtr driver,
>              virDomainShutoffReason reason)
>  {
>      libxlDomainObjPrivatePtr priv = vm->privateData;
> +    virHostdevManagerPtr hostdev_mgr;
>  
>      if (libxl_domain_destroy(priv->ctx, vm->def->id, NULL) < 0) {
>          virReportError(VIR_ERR_INTERNAL_ERROR,
> @@ -749,6 +751,10 @@ libxlVmReap(libxlDriverPrivatePtr driver,
>          return -1;
>      }
>  
> +    hostdev_mgr = virHostdevManagerGetDefault();
> +    virHostdevReAttachDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
> +                                     vm->def, VIR_SP_PCI_HOSTDEV);
> +
>      libxlVmCleanup(driver, vm, reason);
>      return 0;
>  }
> @@ -972,6 +978,7 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
>      char *managed_save_path = NULL;
>      int managed_save_fd = -1;
>      libxlDomainObjPrivatePtr priv = vm->privateData;
> +    virHostdevManagerPtr hostdev_mgr;
>  
>      if (libxlDomainObjPrivateInitCtx(vm) < 0)
>          goto error;
> @@ -1029,6 +1036,12 @@ libxlVmStart(libxlDriverPrivatePtr driver, virDomainObjPtr vm,
>          goto error;
>      }
>  
> +    VIR_DEBUG("Preparing host PCI devices");
> +    hostdev_mgr = virHostdevManagerGetDefault();
> +    if (virHostdevPrepareDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
> +                                        vm->def, VIR_SP_PCI_HOSTDEV) < 0)
> +        goto error;
> +
>      /* use as synchronous operations => ao_how = NULL and no intermediate reports => ao_progress = NULL */
>  
>      if (restore_fd < 0)
> @@ -1122,6 +1135,7 @@ libxlReconnectDomain(virDomainObjPtr vm,
>      libxl_dominfo d_info;
>      int len;
>      uint8_t *data = NULL;
> +    virHostdevManagerPtr hostdev_mgr;
>  
>      virObjectLock(vm);
>  
> @@ -1144,6 +1158,13 @@ libxlReconnectDomain(virDomainObjPtr vm,
>  
>      /* Update domid in case it changed (e.g. reboot) while we were gone? */
>      vm->def->id = d_info.domid;
> +
> +    /* Update hostdev state */
> +    hostdev_mgr = virHostdevManagerGetDefault();
> +    if (virHostdevPrepareDomainHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
> +                                        vm->def, VIR_SP_PCI_HOSTDEV) < 0)
> +        goto out;
> +
>      virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_UNKNOWN);
>  
>      if (!driver->nactive && driver->inhibitCallback)
> @@ -3569,6 +3590,95 @@ cleanup:
>  }
>  
>  static int
> +libxlDomainAttachHostPciDevice(libxlDomainObjPrivatePtr priv,
> +                               virDomainObjPtr vm,
> +                               virDomainHostdevDefPtr hostdev)
> +{
> +    libxl_device_pci pcidev;
> +    virDomainHostdevDefPtr found;
> +    virHostdevManagerPtr hostdev_mgr;
> +
> +    if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
> +        return -1;
> +
> +    if (virDomainHostdevFind(vm->def, hostdev, &found) >= 0) {
> +        virReportError(VIR_ERR_OPERATION_FAILED,
> +                       _("target pci device %.4x:%.2x:%.2x.%.1x already exists"),
> +                       hostdev->source.subsys.u.pci.addr.domain,
> +                       hostdev->source.subsys.u.pci.addr.bus,
> +                       hostdev->source.subsys.u.pci.addr.slot,
> +                       hostdev->source.subsys.u.pci.addr.function);
> +        return -1;
> +    }
> +
> +    if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
> +        return -1;
> +
> +    hostdev_mgr = virHostdevManagerGetDefault();
> +    if (virHostdevPreparePciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
> +                                     vm->def->name, vm->def->uuid,
> +                                     &hostdev, 1, 0) < 0) {
> +        goto cleanup;
> +    }
> +
> +    if (libxlMakePci(hostdev, &pcidev) < 0)
> +        goto reattach_hostdev;
> +
> +    if (libxl_device_pci_add(priv->ctx, vm->def->id, &pcidev, 0) < 0) {
> +        virReportError(VIR_ERR_INTERNAL_ERROR,
> +                       _("libxenlight failed to attach pci device %.4x:%.2x:%.2x.%.1x"),
> +                       hostdev->source.subsys.u.pci.addr.domain,
> +                       hostdev->source.subsys.u.pci.addr.bus,
> +                       hostdev->source.subsys.u.pci.addr.slot,
> +                       hostdev->source.subsys.u.pci.addr.function);
> +        goto reattach_hostdev;
> +    }
> +
> +    vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
> +    return 0;
> +
> +reattach_hostdev:
> +    virHostdevReAttachPciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
> +                                  vm->def->name, &hostdev, 1);
> +
> +cleanup:
> +    return -1;
> +}
> +
> +static int
> +libxlDomainAttachHostDevice(libxlDomainObjPrivatePtr priv,
> +                            virDomainObjPtr vm,
> +                            virDomainDeviceDefPtr dev)
> +{
> +    virDomainHostdevDefPtr hostdev = dev->data.hostdev;
> +
> +    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
> +        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
> +                       _("hostdev mode '%s' not supported"),
> +                       virDomainHostdevModeTypeToString(hostdev->mode));
> +        return -1;
> +    }
> +
> +    switch (hostdev->source.subsys.type) {
> +    case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
> +        if (libxlDomainAttachHostPciDevice(priv, vm, hostdev) < 0)
> +            goto error;
> +        break;
> +
> +    default:
> +        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
> +                       _("hostdev subsys type '%s' not supported"),
> +                       virDomainHostdevSubsysTypeToString(hostdev->source.subsys.type));
> +        goto error;
> +    }
> +
> +    return 0;
> +
> +error:
> +    return -1;
> +}
> +
> +static int
>  libxlDomainDetachDeviceDiskLive(libxlDomainObjPrivatePtr priv,
>                                  virDomainObjPtr vm, virDomainDeviceDefPtr dev)
>  {
> @@ -3635,6 +3745,12 @@ libxlDomainAttachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
>                  dev->data.disk = NULL;
>              break;
>  
> +        case VIR_DOMAIN_DEVICE_HOSTDEV:
> +            ret = libxlDomainAttachHostDevice(priv, vm, dev);
> +            if (!ret)
> +                dev->data.hostdev = NULL;
> +            break;
> +
>          default:
>              virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
>                             _("device type '%s' cannot be attached"),
> @@ -3649,6 +3765,8 @@ static int
>  libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
>  {
>      virDomainDiskDefPtr disk;
> +    virDomainHostdevDefPtr hostdev;
> +    virDomainHostdevDefPtr found;
>  
>      switch (dev->type) {
>          case VIR_DOMAIN_DEVICE_DISK:
> @@ -3663,6 +3781,25 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
>              /* vmdef has the pointer. Generic codes for vmdef will do all jobs */
>              dev->data.disk = NULL;
>              break;
> +        case VIR_DOMAIN_DEVICE_HOSTDEV:
> +            hostdev = dev->data.hostdev;
> +
> +            if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
> +                return -1;
> +
> +            if (virDomainHostdevFind(vmdef, hostdev, &found) >= 0) {
> +                virReportError(VIR_ERR_OPERATION_FAILED,
> +                               _("target pci device %.4x:%.2x:%.2x.%.1x\
> +                                  already exists"),
> +                               hostdev->source.subsys.u.pci.addr.domain,
> +                               hostdev->source.subsys.u.pci.addr.bus,
> +                               hostdev->source.subsys.u.pci.addr.slot,
> +                               hostdev->source.subsys.u.pci.addr.function);
> +                return -1;
> +            }
> +
> +            virDomainHostdevInsert(vmdef, hostdev);
> +            break;
>  
>          default:
>              virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
> @@ -3673,6 +3810,124 @@ libxlDomainAttachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
>  }
>  
>  static int
> +libxlComparePCIDevice(virDomainDefPtr def ATTRIBUTE_UNUSED,
> +                      virDomainDeviceDefPtr device ATTRIBUTE_UNUSED,
> +                      virDomainDeviceInfoPtr info1,
> +                      void *opaque)
> +{
> +    virDomainDeviceInfoPtr info2 = opaque;
> +
> +    if (info1->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI ||
> +        info2->type != VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI)
> +        return 0;
> +
> +    if (info1->addr.pci.domain == info2->addr.pci.domain &&
> +        info1->addr.pci.bus == info2->addr.pci.bus &&
> +        info1->addr.pci.slot == info2->addr.pci.slot &&
> +        info1->addr.pci.function != info2->addr.pci.function)
> +        return -1;
> +    return 0;
> +}
> +
> +static bool
> +libxlIsMultiFunctionDevice(virDomainDefPtr def,
> +                           virDomainDeviceInfoPtr dev)
> +{
> +    if (virDomainDeviceInfoIterate(def, libxlComparePCIDevice, dev) < 0)
> +        return true;
> +    return false;
> +}
> +
> +static int
> +libxlDomainDetachHostPciDevice(libxlDomainObjPrivatePtr priv,
> +                               virDomainObjPtr vm,
> +                               virDomainHostdevDefPtr hostdev)
> +{
> +    virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys;
> +    libxl_device_pci pcidev;
> +    virDomainHostdevDefPtr detach;
> +    int idx;
> +    virHostdevManagerPtr hostdev_mgr;
> +
> +    if (subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI)
> +        return -1;
> +
> +    idx = virDomainHostdevFind(vm->def, hostdev, &detach);
> +    if (idx < 0) {
> +        virReportError(VIR_ERR_OPERATION_FAILED,
> +                       _("host pci device %.4x:%.2x:%.2x.%.1x not found"),
> +                       subsys->u.pci.addr.domain, subsys->u.pci.addr.bus,
> +                       subsys->u.pci.addr.slot, subsys->u.pci.addr.function);
> +        return -1;
> +    }
> +
> +    if (libxlIsMultiFunctionDevice(vm->def, detach->info)) {
> +        virReportError(VIR_ERR_OPERATION_FAILED,
> +                       _("cannot hot unplug multifunction PCI device: %.4x:%.2x:%.2x.%.1x"),
> +                       subsys->u.pci.addr.domain, subsys->u.pci.addr.bus,
> +                       subsys->u.pci.addr.slot, subsys->u.pci.addr.function);
> +        goto cleanup;
> +    }
> +
> +
> +    libxl_device_pci_init(&pcidev);
> +
> +    if (libxlMakePci(detach, &pcidev) < 0)
> +        goto cleanup;
> +
> +    if (libxl_device_pci_remove(priv->ctx, vm->def->id, &pcidev, 0) < 0) {
> +        virReportError(VIR_ERR_INTERNAL_ERROR,
> +                       _("libxenlight failed to detach pci device\
> +                          %.4x:%.2x:%.2x.%.1x"),
> +                       subsys->u.pci.addr.domain, subsys->u.pci.addr.bus,
> +                       subsys->u.pci.addr.slot, subsys->u.pci.addr.function);
> +        goto cleanup;
> +    }
> +
> +    libxl_device_pci_dispose(&pcidev);
> +
> +    virDomainHostdevRemove(vm->def, idx);
> +
> +    hostdev_mgr = virHostdevManagerGetDefault();
> +    virHostdevReAttachPciHostdevs(hostdev_mgr, LIBXL_DRIVER_NAME,
> +                                  vm->def->name, &hostdev, 1);
> +
> +    return 0;
> +
> +cleanup:
> +    virDomainHostdevDefFree(detach);
> +    return -1;
> +}
> +
> +static int
> +libxlDomainDetachHostDevice(libxlDomainObjPrivatePtr priv,
> +                                       virDomainObjPtr vm,
> +                                       virDomainDeviceDefPtr dev)
> +{
> +    virDomainHostdevDefPtr hostdev = dev->data.hostdev;
> +    virDomainHostdevSubsysPtr subsys = &hostdev->source.subsys;
> +
> +    if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS) {
> +        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
> +                       _("hostdev mode '%s' not supported"),
> +                       virDomainHostdevModeTypeToString(hostdev->mode));
> +        return -1;
> +    }
> +
> +    switch (subsys->type) {
> +        case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
> +            return libxlDomainDetachHostPciDevice(priv, vm, hostdev);
> +
> +        default:
> +            virReportError(VIR_ERR_INTERNAL_ERROR,
> +                           _("unexpected hostdev type %d"), subsys->type);
> +            break;
> +    }
> +
> +    return -1;
> +}
> +
> +static int
>  libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
>                              virDomainDeviceDefPtr dev)
>  {
> @@ -3683,6 +3938,10 @@ libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
>              ret = libxlDomainDetachDeviceDiskLive(priv, vm, dev);
>              break;
>  
> +        case VIR_DOMAIN_DEVICE_HOSTDEV:
> +            ret = libxlDomainDetachHostDevice(priv, vm, dev);
> +            break;
> +
>          default:
>              virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
>                             _("device type '%s' cannot be detached"),
> @@ -3693,6 +3952,7 @@ libxlDomainDetachDeviceLive(libxlDomainObjPrivatePtr priv, virDomainObjPtr vm,
>      return ret;
>  }
>  
> +
>  static int
>  libxlDomainDetachDeviceConfig(virDomainDefPtr vmdef, virDomainDeviceDefPtr dev)
>  {
> @@ -4951,7 +5211,7 @@ libxlConnectSupportsFeature(virConnectPtr conn, int feature)
>  
>  static virDriver libxlDriver = {
>      .no = VIR_DRV_LIBXL,
> -    .name = "xenlight",
> +    .name = LIBXL_DRIVER_NAME,
>      .connectOpen = libxlConnectOpen, /* 0.9.0 */
>      .connectClose = libxlConnectClose, /* 0.9.0 */
>      .connectGetType = libxlConnectGetType, /* 0.9.0 */
>   

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list




[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]