Re: [RFC PATCH] libvirt support to force convergence of live guest migration

[Chegu Vinod <chegu_vinod@xxxxxx>]

On 1/29/2014 8:27 AM, Jim Fehlig wrote:
> Chegu Vinod wrote:
> > Busy enterprise workloads hosted on large sized VM's tend to dirty
> > memory faster than the transfer rate achieved via live guest migration.
> > Despite some good recent improvements (& using dedicated 10Gig NICs
> > between hosts) the live migration may NOT converge.
> >
> > Recently support was added in qemu (version 1.6) to allow a user to
> > choose if they wish to force convergence of their migration via a
> > new migration capability : "auto-converge". This feature allows for qemu
> > to auto-detect lack of convergence and trigger a throttle-down of the
> > VCPUs.
> >
> > This RFC patch includes the libvirt support needed to trigger this
> > feature. (Testing is still in progress)
> >    
> Vinod,
>
> What is the status of this patch?  I see there were a few comments, but
> don't see (or perhaps missed) a V2.

Thanks for your follow up query.


> > Signed-off-by:  Chegu Vinod <chegu_vinod@xxxxxx>
> > ---
> >   include/libvirt/libvirt.h.in |    1 +
> >   src/qemu/qemu_migration.c    |   44 ++++++++++++++++++++++++++++++++++++++++++
> >   src/qemu/qemu_migration.h    |    1 +
> >   src/qemu/qemu_monitor.c      |    2 +-
> >   src/qemu/qemu_monitor.h      |    1 +
> >   tools/virsh-domain.c         |    7 ++++++
> >   6 files changed, 55 insertions(+), 1 deletions(-)
> >
> > diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
> > index 146a59b..13b0bfc 100644
> > --- a/include/libvirt/libvirt.h.in
> > +++ b/include/libvirt/libvirt.h.in
> > @@ -1192,6 +1192,7 @@ typedef enum {
> >       VIR_MIGRATE_OFFLINE           = (1 << 10), /* offline migrate */
> >       VIR_MIGRATE_COMPRESSED        = (1 << 11), /* compress data during migration */
> >       VIR_MIGRATE_ABORT_ON_ERROR    = (1 << 12), /* abort migration on I/O errors happened during migration */
> > +    VIR_MIGRATE_AUTO_CONVERGE     = (1 << 13), /* force auto-convergence during during migration */
> >   } virDomainMigrateFlags;
> >   
> >   
> > diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
> > index e87ea85..8cc0c56 100644
> > --- a/src/qemu/qemu_migration.c
> > +++ b/src/qemu/qemu_migration.c
> > @@ -1565,6 +1565,40 @@ cleanup:
> >   }
> >   
> >   static int
> > +qemuMigrationSetAutoConverge(virQEMUDriverPtr driver,
> > +                            virDomainObjPtr vm,
> > +                            enum qemuDomainAsyncJob job)
> > +{
> > +    qemuDomainObjPrivatePtr priv = vm->privateData;
> > +    int ret;
> > +
> > +    if (qemuDomainObjEnterMonitorAsync(driver, vm, job) < 0)
> > +        return -1;
> > +
> > +    ret = qemuMonitorGetMigrationCapability(
> > +                priv->mon,
> > +                QEMU_MONITOR_MIGRATION_CAPS_AUTO_CONVERGE);
> > +
> > +    if (ret < 0) {
> > +        goto cleanup;
> > +    } else if (ret == 0) {
> > +            virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED, "%s",
> > +                           _("Auto-Converge migration is not supported by "
> > +                             "QEMU binary"));
> > +        ret = -1;
> > +        goto cleanup;
> > +    }
> > +
> > +    ret = qemuMonitorSetMigrationCapability(
> > +                priv->mon,
> > +                QEMU_MONITOR_MIGRATION_CAPS_AUTO_CONVERGE);
> > +
> > +cleanup:
> > +    qemuDomainObjExitMonitor(driver, vm);
> > +    return ret;
> > +}
> > +
> > +static int
> >   qemuMigrationWaitForSpice(virQEMUDriverPtr driver,
> >                             virDomainObjPtr vm)
> >   {
> > @@ -2389,6 +2423,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
> >                                       QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
> >           goto stop;
> >   
> > +    if (flags & VIR_MIGRATE_AUTO_CONVERGE &&
> > +        qemuMigrationSetAutoConverge(driver, vm,
> > +                                    QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
> > +        goto stop;
> > +
> >       if (mig->lockState) {
> >           VIR_DEBUG("Received lockstate %s", mig->lockState);
> >           VIR_FREE(priv->lockState);
> > @@ -3181,6 +3220,11 @@ qemuMigrationRun(virQEMUDriverPtr driver,
> >                                       QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
> >           goto cleanup;
> >   
> > +    if (flags & VIR_MIGRATE_AUTO_CONVERGE &&
> > +        qemuMigrationSetAutoConverge(driver, vm,
> > +                                    QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
> > +        goto cleanup;
> > +
> >       if (qemuDomainObjEnterMonitorAsync(driver, vm,
> >                                          QEMU_ASYNC_JOB_MIGRATION_OUT) < 0)
> >           goto cleanup;
> > diff --git a/src/qemu/qemu_migration.h b/src/qemu/qemu_migration.h
> > index cafa2a2..c4258a1 100644
> > --- a/src/qemu/qemu_migration.h
> > +++ b/src/qemu/qemu_migration.h
> > @@ -39,6 +39,7 @@
> >        VIR_MIGRATE_UNSAFE |                       \
> >        VIR_MIGRATE_OFFLINE |                      \
> >        VIR_MIGRATE_COMPRESSED |                   \
> > +     VIR_MIGRATE_AUTO_CONVERGE |                \
> >        VIR_MIGRATE_ABORT_ON_ERROR)
> >   
> >   /* All supported migration parameters and their types. */
> > diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
> > index 1514715..780a29a 100644
> > --- a/src/qemu/qemu_monitor.c
> > +++ b/src/qemu/qemu_monitor.c
> > @@ -118,7 +118,7 @@ VIR_ENUM_IMPL(qemuMonitorMigrationStatus,
> >   
> >   VIR_ENUM_IMPL(qemuMonitorMigrationCaps,
> >                 QEMU_MONITOR_MIGRATION_CAPS_LAST,
> > -              "xbzrle")
> > +              "xbzrle", "auto-converge")
> >   
> >   VIR_ENUM_IMPL(qemuMonitorVMStatus,
> >                 QEMU_MONITOR_VM_STATUS_LAST,
> > diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
> > index eabf000..95e70ab 100644
> > --- a/src/qemu/qemu_monitor.h
> > +++ b/src/qemu/qemu_monitor.h
> > @@ -440,6 +440,7 @@ int qemuMonitorGetSpiceMigrationStatus(qemuMonitorPtr mon,
> >   
> >   typedef enum {
> >       QEMU_MONITOR_MIGRATION_CAPS_XBZRLE,
> > +    QEMU_MONITOR_MIGRATION_CAPS_AUTO_CONVERGE,
> >   
> >       QEMU_MONITOR_MIGRATION_CAPS_LAST
> >   } qemuMonitorMigrationCaps;
> > diff --git a/tools/virsh-domain.c b/tools/virsh-domain.c
> > index 1fe138c..d94a81b 100644
> > --- a/tools/virsh-domain.c
> > +++ b/tools/virsh-domain.c
> > @@ -8532,6 +8532,10 @@ static const vshCmdOptDef opts_migrate[] = {
> >        .type = VSH_OT_BOOL,
> >        .help = N_("compress repeated pages during live migration")
> >       },
> > +    {.name = "auto-converge",
> > +     .type = VSH_OT_BOOL,
> > +     .help = N_("force auto convergence during live migration")
> > +    },
> >       {.name = "abort-on-error",
> >        .type = VSH_OT_BOOL,
> >        .help = N_("abort on soft errors during migration")
> > @@ -8676,6 +8680,9 @@ doMigrate(void *opaque)
> >       if (vshCommandOptBool(cmd, "compressed"))
> >           flags |= VIR_MIGRATE_COMPRESSED;
> >   
> > +    if (vshCommandOptBool(cmd, "auto-converge"))
> > +        flags |= VIR_MIGRATE_AUTO_CONVERGE;
> > +
> >       if (vshCommandOptBool(cmd, "offline")) {
> >           flags |= VIR_MIGRATE_OFFLINE;
> >       }
> >    
> virsh.pod needs updated too.
>
> Do you have time to continue working on this patch?  If not, I could
> find some cycles to help out.  Thanks!
Sorry for not responding...I have been busy with other things.

There were some minor comments on this specific patch. I can work with 
you and address those soon.

But there is also another comment about having a new virsh interface to 
allow for this capability to be changed during migration. I did look 
into that but it requires a lot more changes in both libvirt and qemu 
(qemu currently doesn't seem to support changing migration capabilities 
once the migration has started... so we may have to defer that for now).

Thanks
Vinod

> Regards,
> Jim
> .

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list