On 09/17/2014 10:53 AM, Jiri Denemark wrote: > From: "Michael R. Hines" <mrhines@xxxxxxxxxx> > > This patch adds support for RDMA protocol in migration URIs. > > USAGE: $ virsh migrate --live --migrateuri rdma://hostname domain qemu+ssh://hostname/system > > Since libvirt runs QEMU in a pretty restricted environment, several > files needs to be added to cgroup_device_acl (in qemu.conf) for QEMU to > be able to access the host's infiniband hardware. Full documenation of > the feature can be found on QEMU wiki: > http://wiki.qemu.org/Features/RDMALiveMigration > > Signed-off-by: Michael R. Hines <mrhines@xxxxxxxxxx> > Signed-off-by: Jiri Denemark <jdenemar@xxxxxxxxxx> > --- > > Notes: > The question is whether the IB devices should be added to > cgroup_device_acl by default or not... > > Version 3: > - moved capabilities code into a separate patch > - got rid of migration URI hacks > - removed hacks that disabled IPv6 with RDMA > - moved refactoring into a dedicated patch > - documented IB devices which need to be added to cgroup acl in > qemu.conf > - forbid RDMA migrations unless memory hard limit is set until we > have a better plan for setting limits for mlock > - set QEMU's RLIMIT_MEMLOCK to memory hard_limit before starting > RDMA migration > - check if RDMA migration is supported by source QEMU before trying > to migrate > > src/qemu/qemu.conf | 8 ++++++++ > src/qemu/qemu_command.c | 8 ++++++++ > src/qemu/qemu_migration.c | 39 +++++++++++++++++++++++++++++++++++++-- > 3 files changed, 53 insertions(+), 2 deletions(-) > My morning Coverity run found a memory leak... > diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf > index 79bba36..92ca715 100644 > --- a/src/qemu/qemu.conf > +++ b/src/qemu/qemu.conf > @@ -274,6 +274,14 @@ > # "/dev/ptmx", "/dev/kvm", "/dev/kqemu", > # "/dev/rtc","/dev/hpet", "/dev/vfio/vfio" > #] > +# > +# RDMA migration requires the following extra files to be added to the list: > +# "/dev/infiniband/rdma_cm", > +# "/dev/infiniband/issm0", > +# "/dev/infiniband/issm1", > +# "/dev/infiniband/umad0", > +# "/dev/infiniband/umad1", > +# "/dev/infiniband/uverbs0" > > > # The default format for Qemu/KVM guest save images is raw; that is, the > diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c > index a892d99..fceed62 100644 > --- a/src/qemu/qemu_command.c > +++ b/src/qemu/qemu_command.c > @@ -9399,6 +9399,14 @@ qemuBuildCommandLine(virConnectPtr conn, > goto error; > } > virCommandAddArg(cmd, migrateFrom); > + } else if (STRPREFIX(migrateFrom, "rdma")) { > + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) { > + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", > + _("incoming RDMA migration is not supported " > + "with this QEMU binary")); > + goto error; > + } > + virCommandAddArg(cmd, migrateFrom); > } else if (STREQ(migrateFrom, "stdio")) { > if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) { > virCommandAddArgFormat(cmd, "fd:%d", migrateFd); > diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c > index d0e2653..b59e94d 100644 > --- a/src/qemu/qemu_migration.c > +++ b/src/qemu/qemu_migration.c > @@ -56,6 +56,7 @@ > #include "virhook.h" > #include "virstring.h" > #include "virtypedparam.h" > +#include "virprocess.h" > > #define VIR_FROM_THIS VIR_FROM_QEMU > > @@ -2653,6 +2654,13 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, > QEMU_MIGRATION_COOKIE_NBD))) > goto cleanup; > > + if (STREQ(protocol, "rdma") && !vm->def->mem.hard_limit) { > + virReportError(VIR_ERR_OPERATION_INVALID, "%s", > + _("cannot start RDMA migration with no memory hard " > + "limit set")); > + goto cleanup; > + } > + > if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0) > goto cleanup; > qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE); > @@ -2696,6 +2704,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver, > QEMU_ASYNC_JOB_MIGRATION_IN) < 0) > goto stop; > > + if (STREQ(protocol, "rdma") && > + virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) { > + goto stop; > + } > + > if (mig->lockState) { > VIR_DEBUG("Received lockstate %s", mig->lockState); > VIR_FREE(priv->lockState); > @@ -2926,7 +2939,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver, > if (!(uri = qemuMigrationParseURI(uri_in, &well_formed_uri))) > goto cleanup; > > - if (STRNEQ(uri->scheme, "tcp")) { > + if (STRNEQ(uri->scheme, "tcp") && > + STRNEQ(uri->scheme, "rdma")) { > virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, > _("unsupported scheme %s in migration URI %s"), > uri->scheme, uri_in); > @@ -3545,6 +3559,11 @@ qemuMigrationRun(virQEMUDriverPtr driver, > > switch (spec->destType) { > case MIGRATION_DEST_HOST: > + if (STREQ(spec->dest.host.protocol, "rdma") && > + virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) { > + qemuDomainObjExitMonitor(driver, vm); > + goto cleanup; > + } > ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags, > spec->dest.host.protocol, > spec->dest.host.name, > @@ -3717,7 +3736,23 @@ static int doNativeMigrate(virQEMUDriverPtr driver, > if (!(uribits = qemuMigrationParseURI(uri, NULL))) > return -1; > > - if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) > + if (STREQ(uribits->scheme, "rdma")) { > + if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) { > + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", > + _("outgoing RDMA migration is not supported " > + "with this QEMU binary")); Need to virURIFree(uribits); or add a cleanup label below and jump there > + return -1; > + } > + if (!vm->def->mem.hard_limit) { > + virReportError(VIR_ERR_OPERATION_INVALID, "%s", > + _("cannot start RDMA migration with no memory hard " > + "limit set")); > + return -1; Same here > + } > + } > + > + if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) && > + STRNEQ(uribits->scheme, "rdma")) > spec.destType = MIGRATION_DEST_CONNECT_HOST; > else > spec.destType = MIGRATION_DEST_HOST; > -- libvir-list mailing list libvir-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/libvir-list