All,
Attached is the current version of the tunnelled migration patch, based
upon danpb's generic datastream work. In order to use this work, you must first
grab danpb's data-streams git branch here:
http://gitorious.org/~berrange/libvirt/staging
and then apply this patch on top.
In some basic testing, this seems to work fine for me, although I have not given
it a difficult scenario nor measured CPU utilization with these patches in place.
DanB, these patches take a slightly different approach than you and I
discussed yesterday on IRC. Just to recap, you suggested a new version of
virMigratePrepare (called virMigratePrepareTunnel) that would take in as one of
the arguments a datastream, and during the prepare step properly setup the
datastream. Unless I'm missing something (which is entirely possible), this
would also require passing that same datastream into the perform and finish
stages, meaning that I'd essentially have an all new migration protocol version 3.
To try to avoid that, during the prepare I store the port that we used to
start the listening qemu in a new field in the virDomainObj structure. Then
during the perform step, I create a datastream on the destination and run a new
RPC function called virDomainMigratePrepareTunnel. This looks that port back
up, associates it with the current stream, and returns back to the caller. Then
the source side just does virStreamSend for all the data, and we have tunnelled
migration.
TODO:
- More testing, especially under worst-case scenarios (VM constantly
changing it's memory during migration)
- CPU utilization testing to make sure that we aren't using a lot of CPU
time doing this
- Wall-clock testing
- Switch over to using Unix Domain Sockets instead of localhost TCP
migration. With a patch I put into upstream qemu (and is now in F-12), we can
totally get rid of the scanning of localhost ports to find a free one, and just
use Unix Domain Sockets. That should make the whole thing more robust.
--
Chris Lalancette
commit f478a8993ed29c489c2cba7dfd56fd27f92f5518
Author: Chris Lalancette <clalance@xxxxxxxxxx>
Date: Thu Jul 16 10:33:09 2009 +0200
Tunnelled migration.
In progress patch to do tunnelled migration using the virStream stuff from
danpb. Compile and lightly runtime tested so far, seems to be doing the
trick.
Signed-off-by: Chris Lalancette <clalance@xxxxxxxxxx>
diff --git a/docs/apibuild.py b/docs/apibuild.py
index 84bc1ac..e0cb6f8 100755
--- a/docs/apibuild.py
+++ b/docs/apibuild.py
@@ -38,6 +38,7 @@ ignored_functions = {
"virDomainMigratePerform": "private function for migration",
"virDomainMigratePrepare": "private function for migration",
"virDomainMigratePrepare2": "private function for migration",
+ "virDomainMigratePrepareTunnel": "private function for tunnelled migration",
"virDrvSupportsFeature": "private function for remote access",
"DllMain": "specific function for Win32",
}
diff --git a/include/libvirt/libvirt.h b/include/libvirt/libvirt.h
index 96d8c4d..0767d84 100644
--- a/include/libvirt/libvirt.h
+++ b/include/libvirt/libvirt.h
@@ -337,6 +337,7 @@ typedef virDomainInterfaceStatsStruct *virDomainInterfaceStatsPtr;
/* Domain migration flags. */
typedef enum {
VIR_MIGRATE_LIVE = 1, /* live migration */
+ VIR_MIGRATE_TUNNELLED = 2, /* tunnelled migration */
} virDomainMigrateFlags;
/* Domain migration. */
diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
index b73bd59..2680026 100644
--- a/include/libvirt/libvirt.h.in
+++ b/include/libvirt/libvirt.h.in
@@ -337,6 +337,7 @@ typedef virDomainInterfaceStatsStruct *virDomainInterfaceStatsPtr;
/* Domain migration flags. */
typedef enum {
VIR_MIGRATE_LIVE = 1, /* live migration */
+ VIR_MIGRATE_TUNNELLED = 2, /* tunnelled migration */
} virDomainMigrateFlags;
/* Domain migration. */
diff --git a/qemud/dispatch.c b/qemud/dispatch.c
index bd52bcf..838f30f 100644
--- a/qemud/dispatch.c
+++ b/qemud/dispatch.c
@@ -546,7 +546,6 @@ fatal_error:
}
-#if 0
static int
remoteSendStreamData(struct qemud_client *client,
struct qemud_client_stream *stream,
@@ -622,7 +621,6 @@ fatal_error:
VIR_FREE(msg);
return -1;
}
-#endif
/*
diff --git a/qemud/remote.c b/qemud/remote.c
index c161982..bd44df9 100644
--- a/qemud/remote.c
+++ b/qemud/remote.c
@@ -1490,6 +1490,39 @@ remoteDispatchDomainMigrateFinish2 (struct qemud_server *server ATTRIBUTE_UNUSED
}
static int
+remoteDispatchDomainMigratePrepareTunnel (struct qemud_server *server ATTRIBUTE_UNUSED,
+ struct qemud_client *client,
+ virConnectPtr conn,
+ remote_message_header *hdr,
+ remote_error *rerr,
+ remote_domain_migrate_prepare_tunnel_args *args,
+ void *ret ATTRIBUTE_UNUSED)
+{
+ struct qemud_client_stream *stream;
+ int r;
+ CHECK_CONN (client);
+
+ stream = remoteCreateClientStream(conn, hdr);
+ if (!stream) {
+ remoteDispatchOOMError(rerr);
+ return -1;
+ }
+
+ r = virDomainMigratePrepareTunnel (conn, stream->st,
+ (unsigned char *)args->uuid,
+ args->flags);
+ if (r == -1) {
+ remoteFreeClientStream(stream);
+ remoteDispatchConnError(rerr, conn);
+ return -1;
+ }
+
+ remoteAddClientStream(client, stream);
+
+ return 0;
+}
+
+static int
remoteDispatchListDefinedDomains (struct qemud_server *server ATTRIBUTE_UNUSED,
struct qemud_client *client ATTRIBUTE_UNUSED,
virConnectPtr conn,
diff --git a/qemud/remote_dispatch_args.h b/qemud/remote_dispatch_args.h
index 53126c2..5a9a677 100644
--- a/qemud/remote_dispatch_args.h
+++ b/qemud/remote_dispatch_args.h
@@ -118,3 +118,4 @@
remote_domain_xml_to_native_args val_remote_domain_xml_to_native_args;
remote_put_file_args val_remote_put_file_args;
remote_get_file_args val_remote_get_file_args;
+ remote_domain_migrate_prepare_tunnel_args val_remote_domain_migrate_prepare_tunnel_args;
diff --git a/qemud/remote_dispatch_prototypes.h b/qemud/remote_dispatch_prototypes.h
index 7529afc..c29f935 100644
--- a/qemud/remote_dispatch_prototypes.h
+++ b/qemud/remote_dispatch_prototypes.h
@@ -1098,3 +1098,11 @@ static int remoteDispatchSupportsFeature(
remote_error *err,
remote_supports_feature_args *args,
remote_supports_feature_ret *ret);
+static int remoteDispatchDomainMigratePrepareTunnel(
+ struct qemud_server *server,
+ struct qemud_client *client,
+ virConnectPtr conn,
+ remote_message_header *hdr,
+ remote_error *err,
+ remote_domain_migrate_prepare_tunnel_args *args,
+ void *ret);
diff --git a/qemud/remote_dispatch_table.h b/qemud/remote_dispatch_table.h
index 6f6fc58..d238edd 100644
--- a/qemud/remote_dispatch_table.h
+++ b/qemud/remote_dispatch_table.h
@@ -697,3 +697,8 @@
.args_filter = (xdrproc_t) xdr_remote_get_file_args,
.ret_filter = (xdrproc_t) xdr_void,
},
+{ /* DomainMigratePrepareTunnel => 139 */
+ .fn = (dispatch_fn) remoteDispatchDomainMigratePrepareTunnel,
+ .args_filter = (xdrproc_t) xdr_remote_domain_migrate_prepare_tunnel_args,
+ .ret_filter = (xdrproc_t) xdr_void,
+},
diff --git a/qemud/remote_protocol.c b/qemud/remote_protocol.c
index 816dc97..bf183a8 100644
--- a/qemud/remote_protocol.c
+++ b/qemud/remote_protocol.c
@@ -2523,6 +2523,17 @@ xdr_remote_get_file_args (XDR *xdrs, remote_get_file_args *objp)
}
bool_t
+xdr_remote_domain_migrate_prepare_tunnel_args (XDR *xdrs, remote_domain_migrate_prepare_tunnel_args *objp)
+{
+
+ if (!xdr_remote_nonnull_string (xdrs, &objp->uuid))
+ return FALSE;
+ if (!xdr_uint64_t (xdrs, &objp->flags))
+ return FALSE;
+ return TRUE;
+}
+
+bool_t
xdr_remote_procedure (XDR *xdrs, remote_procedure *objp)
{
diff --git a/qemud/remote_protocol.h b/qemud/remote_protocol.h
index ded5b16..dade228 100644
--- a/qemud/remote_protocol.h
+++ b/qemud/remote_protocol.h
@@ -1418,6 +1418,12 @@ struct remote_get_file_args {
remote_nonnull_string name;
};
typedef struct remote_get_file_args remote_get_file_args;
+
+struct remote_domain_migrate_prepare_tunnel_args {
+ remote_nonnull_string uuid;
+ uint64_t flags;
+};
+typedef struct remote_domain_migrate_prepare_tunnel_args remote_domain_migrate_prepare_tunnel_args;
#define REMOTE_PROGRAM 0x20008086
#define REMOTE_PROTOCOL_VERSION 1
@@ -1560,6 +1566,7 @@ enum remote_procedure {
REMOTE_PROC_DOMAIN_XML_TO_NATIVE = 136,
REMOTE_PROC_PUT_FILE = 137,
REMOTE_PROC_GET_FILE = 138,
+ REMOTE_PROC_DOMAIN_MIGRATE_PREPARE_TUNNEL = 139,
};
typedef enum remote_procedure remote_procedure;
@@ -1823,6 +1830,7 @@ extern bool_t xdr_remote_domain_xml_to_native_args (XDR *, remote_domain_xml_to
extern bool_t xdr_remote_domain_xml_to_native_ret (XDR *, remote_domain_xml_to_native_ret*);
extern bool_t xdr_remote_put_file_args (XDR *, remote_put_file_args*);
extern bool_t xdr_remote_get_file_args (XDR *, remote_get_file_args*);
+extern bool_t xdr_remote_domain_migrate_prepare_tunnel_args (XDR *, remote_domain_migrate_prepare_tunnel_args*);
extern bool_t xdr_remote_procedure (XDR *, remote_procedure*);
extern bool_t xdr_remote_message_type (XDR *, remote_message_type*);
extern bool_t xdr_remote_message_status (XDR *, remote_message_status*);
@@ -2060,6 +2068,7 @@ extern bool_t xdr_remote_domain_xml_to_native_args ();
extern bool_t xdr_remote_domain_xml_to_native_ret ();
extern bool_t xdr_remote_put_file_args ();
extern bool_t xdr_remote_get_file_args ();
+extern bool_t xdr_remote_domain_migrate_prepare_tunnel_args ();
extern bool_t xdr_remote_procedure ();
extern bool_t xdr_remote_message_type ();
extern bool_t xdr_remote_message_status ();
diff --git a/qemud/remote_protocol.x b/qemud/remote_protocol.x
index 428489b..46932e7 100644
--- a/qemud/remote_protocol.x
+++ b/qemud/remote_protocol.x
@@ -1259,6 +1259,11 @@ struct remote_get_file_args {
remote_nonnull_string name;
};
+struct remote_domain_migrate_prepare_tunnel_args {
+ remote_nonnull_string uuid;
+ unsigned hyper flags;
+};
+
/*----- Protocol. -----*/
/* Define the program number, protocol version and procedure numbers here. */
@@ -1417,7 +1422,9 @@ enum remote_procedure {
REMOTE_PROC_DOMAIN_XML_TO_NATIVE = 136,
REMOTE_PROC_PUT_FILE = 137,
- REMOTE_PROC_GET_FILE = 138
+ REMOTE_PROC_GET_FILE = 138,
+
+ REMOTE_PROC_DOMAIN_MIGRATE_PREPARE_TUNNEL = 139
};
diff --git a/src/domain_conf.h b/src/domain_conf.h
index 6e111fa..d5cb643 100644
--- a/src/domain_conf.h
+++ b/src/domain_conf.h
@@ -542,6 +542,7 @@ struct _virDomainObj {
int monitorWatch;
int pid;
int state;
+ int incoming;
int nvcpupids;
int *vcpupids;
diff --git a/src/driver.h b/src/driver.h
index 851c3a6..7d12aa0 100644
--- a/src/driver.h
+++ b/src/driver.h
@@ -346,6 +346,13 @@ typedef int
typedef int (*virDrvPutFile)(virConnectPtr conn, const char *name, virStreamPtr st);
typedef int (*virDrvGetFile)(virConnectPtr conn, const char *name, virStreamPtr st);
+typedef int
+ (*virDrvDomainMigratePrepareTunnel)
+ (virConnectPtr conn,
+ virStreamPtr st,
+ const unsigned char *uuid,
+ unsigned long flags);
+
/**
* _virDriver:
*
@@ -428,6 +435,7 @@ struct _virDriver {
virDrvNodeDeviceReset nodeDeviceReset;
virDrvPutFile putFile;
virDrvGetFile getFile;
+ virDrvDomainMigratePrepareTunnel domainMigratePrepareTunnel;
};
typedef int
diff --git a/src/libvirt.c b/src/libvirt.c
index f0be69e..2aa6ef9 100644
--- a/src/libvirt.c
+++ b/src/libvirt.c
@@ -2852,6 +2852,8 @@ error:
*
* Flags may be one of more of the following:
* VIR_MIGRATE_LIVE Attempt a live migration.
+ * VIR_MIGRATE_TUNNELLED Attempt to do a migration tunnelled through the
+ * libvirt RPC mechanism
*
* If a hypervisor supports renaming domains during migration,
* then you may set the dname parameter to the new name (otherwise
@@ -2925,6 +2927,13 @@ virDomainMigrate (virDomainPtr domain,
goto error;
}
+ if ((flags & VIR_MIGRATE_TUNNELLED) && uri == NULL) {
+ /* if you are doing a secure migration, you *must* also pass a uri */
+ virLibConnError(conn, VIR_ERR_INVALID_ARG,
+ _("requested TUNNELLED migration, but no URI passed"));
+ goto error;
+ }
+
/* Check that migration is supported by both drivers. */
if (VIR_DRV_SUPPORTS_FEATURE (conn->driver, conn,
VIR_DRV_FEATURE_MIGRATION_V1) &&
@@ -3042,13 +3051,13 @@ error:
*/
int
virDomainMigratePrepare (virConnectPtr dconn,
- char **cookie,
- int *cookielen,
- const char *uri_in,
- char **uri_out,
- unsigned long flags,
- const char *dname,
- unsigned long bandwidth)
+ char **cookie,
+ int *cookielen,
+ const char *uri_in,
+ char **uri_out,
+ unsigned long flags,
+ const char *dname,
+ unsigned long bandwidth)
{
VIR_DEBUG("dconn=%p, cookie=%p, cookielen=%p, uri_in=%s, uri_out=%p, "
"flags=%lu, dname=%s, bandwidth=%lu", dconn, cookie, cookielen,
@@ -3279,6 +3288,50 @@ error:
return NULL;
}
+/*
+ * Not for public use. This function is part of the internal
+ * implementation of migration in the remote case.
+ */
+int
+virDomainMigratePrepareTunnel (virConnectPtr conn,
+ virStreamPtr st,
+ const unsigned char *uuid,
+ unsigned long flags)
+{
+ VIR_DEBUG("conn=%p, uuid=%s, flags=%lu", conn, uuid, flags);
+
+ virResetLastError();
+
+ if (!VIR_IS_CONNECT(conn)) {
+ virLibConnError(NULL, VIR_ERR_INVALID_CONN, __FUNCTION__);
+ return -1;
+ }
+
+ if (conn->flags & VIR_CONNECT_RO) {
+ virLibConnError(conn, VIR_ERR_OPERATION_DENIED, __FUNCTION__);
+ goto error;
+ }
+
+ if (conn != st->conn) {
+ virLibConnError(conn, VIR_ERR_INVALID_ARG, __FUNCTION__);
+ goto error;
+ }
+
+ if (conn->driver->domainMigratePrepareTunnel) {
+ int rv = conn->driver->domainMigratePrepareTunnel (conn, st,
+ uuid, flags);
+ if (rv < 0)
+ goto error;
+ return rv;
+ }
+
+ virLibConnError (conn, VIR_ERR_NO_SUPPORT, __FUNCTION__);
+
+error:
+ /* Copy to connection error object for back compatability */
+ virSetConnError(conn);
+ return -1;
+}
/**
* virNodeGetInfo:
@@ -8558,7 +8611,7 @@ virStreamRef(virStreamPtr stream)
* Write a series of bytes to the stream. This method may
* block the calling application for an arbitrary amount
* of time. Once an application has finished sending data
- * it should call virStreamFinish to wait for succesful
+ * it should call virStreamFinish to wait for successful
* confirmation from the driver, or detect any error
*
* This method may not be used if a stream source has been
@@ -8813,11 +8866,11 @@ error:
/**
- * virStreamAbort:
+ * virStreamFinish:
* @stream: pointer to the stream object
*
- * Request that the in progress data transfer be cancelled
- * abnormally before the end of the stream has been reached.
+ * Request that the data transfer be completed, and any errors on the
+ * stream be returned to the caller.
*
* Returns 0 on success, -1 upon error
*/
diff --git a/src/libvirt_internal.h b/src/libvirt_internal.h
index 8800eb9..0bb16bc 100644
--- a/src/libvirt_internal.h
+++ b/src/libvirt_internal.h
@@ -72,5 +72,9 @@ virDomainPtr virDomainMigrateFinish2 (virConnectPtr dconn,
unsigned long flags,
int retcode);
+int virDomainMigratePrepareTunnel (virConnectPtr conn,
+ virStreamPtr st,
+ const unsigned char *uuid,
+ unsigned long flags);
#endif
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 0534d53..0286326 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -54,6 +54,10 @@ virUnrefStorageVol;
virGetNodeDevice;
virUnrefDomain;
virUnrefConnect;
+virGetStream;
+virUnrefStream;
+virStreamSendAll;
+virStreamRecvAll;
# domain_conf.h
@@ -198,6 +202,7 @@ virDomainMigratePerform;
virDomainMigrateFinish;
virDomainMigratePrepare2;
virDomainMigrateFinish2;
+virDomainMigratePrepareTunnel;
virRegisterDriver;
virRegisterNetworkDriver;
virRegisterStateDriver;
diff --git a/src/lxc_driver.c b/src/lxc_driver.c
index 83f0118..2a51a99 100644
--- a/src/lxc_driver.c
+++ b/src/lxc_driver.c
@@ -1499,6 +1499,7 @@ static virDriver lxcDriver = {
NULL, /* nodeDeviceReset */
NULL,
NULL,
+ NULL, /* domainMigratePrepareTunnel */
};
static virStateDriver lxcStateDriver = {
diff --git a/src/opennebula/one_driver.c b/src/opennebula/one_driver.c
index 184720b..1d71929 100644
--- a/src/opennebula/one_driver.c
+++ b/src/opennebula/one_driver.c
@@ -780,7 +780,8 @@ static virDriver oneDriver = {
NULL, /* nodeDeviceReAttach; */
NULL, /* nodeDeviceReset; */
NULL,
- NULL
+ NULL,
+ NULL, /* domainMigratePrepareTunnel */
};
static virStateDriver oneStateDriver = {
diff --git a/src/openvz_driver.c b/src/openvz_driver.c
index ca718d1..e43d088 100644
--- a/src/openvz_driver.c
+++ b/src/openvz_driver.c
@@ -1394,6 +1394,7 @@ static virDriver openvzDriver = {
NULL, /* nodeDeviceReset */
NULL,
NULL,
+ NULL, /* domainMigratePrepareTunnel */
};
int openvzRegister(void) {
diff --git a/src/qemu_driver.c b/src/qemu_driver.c
index 815bb24..ae7d9be 100644
--- a/src/qemu_driver.c
+++ b/src/qemu_driver.c
@@ -66,6 +66,7 @@
#include "node_device_conf.h"
#include "pci.h"
#include "security.h"
+#include "libvirt_internal.h"
#define VIR_FROM_THIS VIR_FROM_QEMU
@@ -4913,7 +4914,7 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn,
int *cookielen ATTRIBUTE_UNUSED,
const char *uri_in,
char **uri_out,
- unsigned long flags ATTRIBUTE_UNUSED,
+ unsigned long flags,
const char *dname,
unsigned long resource ATTRIBUTE_UNUSED,
const char *dom_xml)
@@ -4927,7 +4928,7 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn,
char migrateFrom [64];
const char *p;
virDomainEventPtr event = NULL;
- int ret = -1;;
+ int ret = -1;
*uri_out = NULL;
@@ -4947,6 +4948,8 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn,
* If the URI passed in is not NULL then we try to parse out the
* port number and use that (note that the hostname is assumed
* to be a correct hostname which refers to the target machine).
+ *
+ * Note that for tunnelled migration, uri_in is guaranteed to be non-null
*/
if (uri_in == NULL) {
this_port = QEMUD_MIGRATION_FIRST_PORT + port++;
@@ -4965,24 +4968,48 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn,
goto cleanup;
}
} else {
- /* Check the URI starts with "tcp:". We will escape the
- * URI when passing it to the qemu monitor, so bad
- * characters in hostname part don't matter.
- */
- if (!STRPREFIX (uri_in, "tcp:")) {
- qemudReportError (dconn, NULL, NULL, VIR_ERR_INVALID_ARG,
- "%s", _("only tcp URIs are supported for KVM migrations"));
- goto cleanup;
+ if (!(flags & VIR_MIGRATE_TUNNELLED)) {
+ /* Check the URI starts with "tcp:". We will escape the
+ * URI when passing it to the qemu monitor, so bad
+ * characters in hostname part don't matter.
+ */
+ if (!STRPREFIX (uri_in, "tcp:")) {
+ qemudReportError (dconn, NULL, NULL, VIR_ERR_INVALID_ARG,
+ "%s", _("only tcp URIs are supported for Qemu migrations"));
+ goto cleanup;
+ }
+
+ /* Get the port number. */
+ p = strrchr (uri_in, ':');
+ p++; /* definitely has a ':' in it, see above */
+ this_port = virParseNumber (&p);
+ if (this_port == -1 || p-uri_in != strlen (uri_in)) {
+ qemudReportError (dconn, NULL, NULL, VIR_ERR_INVALID_ARG,
+ "%s", _("URI did not have ':port' at the end"));
+ goto cleanup;
+ }
}
+ else {
+ /* Tunnelled migration requested; find a free port */
+ /* FIXME: this is kind of silly. What we are really trying to do
+ * here is find any open port locally that we can use to migrate
+ * the guest over localhost. So we should be able to go through
+ * all ports on the host and look. Even better, if we get away
+ * from tcp migrations and instead use exec migrations with nc for
+ * UDP domain sockets, we get rid of the port problem completely
+ */
+ this_port = QEMUD_MIGRATION_FIRST_PORT + port++;
+ if (port == QEMUD_MIGRATION_NUM_PORTS) port = 0;
- /* Get the port number. */
- p = strrchr (uri_in, ':');
- p++; /* definitely has a ':' in it, see above */
- this_port = virParseNumber (&p);
- if (this_port == -1 || p-uri_in != strlen (uri_in)) {
- qemudReportError (dconn, NULL, NULL, VIR_ERR_INVALID_ARG,
- "%s", _("URI did not have ':port' at the end"));
- goto cleanup;
+ /* for tunnelled migration, uri_out is more or less meaningless
+ * (since we are migrating to localhost anyway). Dup uri_in to
+ * maintain our contract
+ */
+ *uri_out = strdup(uri_in);
+ if (*uri_out == NULL) {
+ virReportOOMError(NULL);
+ goto cleanup;
+ }
}
}
@@ -5038,8 +5065,14 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn,
/* Start the QEMU daemon, with the same command-line arguments plus
* -incoming tcp:0.0.0.0:port
*/
- snprintf (migrateFrom, sizeof (migrateFrom), "tcp:0.0.0.0:%d", this_port);
+ if (!(flags & VIR_MIGRATE_TUNNELLED))
+ snprintf (migrateFrom, sizeof (migrateFrom), "tcp:0.0.0.0:%d", this_port);
+ else
+ snprintf (migrateFrom, sizeof (migrateFrom), "tcp:127.0.0.1:%d", this_port);
+
if (qemudStartVMDaemon (dconn, driver, vm, migrateFrom, -1) < 0) {
+ qemudReportError (dconn, NULL, NULL, VIR_ERR_OPERATION_FAILED,
+ "%s", _("failed to start listening VM"));
if (!vm->persistent) {
virDomainRemoveInactive(&driver->domains, vm);
vm = NULL;
@@ -5050,12 +5083,24 @@ qemudDomainMigratePrepare2 (virConnectPtr dconn,
event = virDomainEventNewFromObj(vm,
VIR_DOMAIN_EVENT_STARTED,
VIR_DOMAIN_EVENT_STARTED_MIGRATED);
+
+ /* remember the port we used; this is used for tunnelled migration, so
+ * we can later set up the stream correctly
+ */
+ vm->incoming = this_port;
+
ret = 0;
cleanup:
virDomainDefFree(def);
if (ret != 0) {
VIR_FREE(*uri_out);
+
+ /* there might have been an error after we started the incoming qemu
+ * process, so be sure to kill it before we leave
+ */
+ if (vm)
+ qemudShutdownVMDaemon(NULL, driver, vm);
}
if (vm)
virDomainObjUnlock(vm);
@@ -5065,13 +5110,277 @@ cleanup:
return ret;
}
+static int
+qemuMigCloseSock(virStreamPtr st)
+{
+ int *qemusock = st->privateData;
+ struct qemud_driver *driver = st->conn->privateData;
+
+ qemuDriverLock(driver);
+ close(*qemusock);
+ VIR_FREE(qemusock);
+ qemuDriverUnlock(driver);
+
+ return 0;
+}
+
+static int
+qemuMigWrite(virStreamPtr st,
+ const char *bytes,
+ size_t nbytes)
+{
+ int *qemusock = st->privateData;
+ struct qemud_driver *driver = st->conn->privateData;
+ int ret;
+
+ qemuDriverLock(driver);
+ ret = safewrite(*qemusock, bytes, nbytes);
+ qemuDriverUnlock(driver);
+
+ return ret;
+}
+
+static virStreamDriver migrateFileDrv = {
+ .streamSend = qemuMigWrite,
+ .streamFinish = qemuMigCloseSock,
+ .streamAbort = qemuMigCloseSock,
+};
+
+static int
+qemudDomainMigratePrepareTunnel (virConnectPtr conn,
+ virStreamPtr st,
+ const unsigned char *uuid,
+ unsigned long flags ATTRIBUTE_UNUSED)
+{
+ struct qemud_driver *driver = conn->privateData;
+ virDomainObjPtr vm;
+ int ret = -1;
+ int *qemusock = NULL;
+ struct sockaddr_in a;
+
+ qemuDriverLock(driver);
+ vm = virDomainFindByUUID(&driver->domains, uuid);
+ if (!vm) {
+ qemudReportError (conn, NULL, NULL, VIR_ERR_OPERATION_FAILED,
+ "%s", _("failed to find VM"));
+ goto cleanup;
+ }
+
+ if (VIR_ALLOC(qemusock) < 0) {
+ virReportOOMError (conn);
+ goto cleanup;
+ }
+
+ /* here, we need to set up our sink for the incoming data */
+ *qemusock = socket(AF_INET, SOCK_STREAM, 0);
+ if (*qemusock < 0) {
+ virReportSystemError(conn, errno,
+ "%s", _("cannot open socket"));
+ VIR_FREE(qemusock);
+ goto cleanup;
+ }
+
+ memset(&a, 0, sizeof(a));
+ a.sin_port = htons(vm->incoming);
+ a.sin_family = AF_INET;
+ a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+
+ if (connect(*qemusock, (struct sockaddr *)&a, sizeof(a)) < 0) {
+ virReportSystemError(conn, errno,
+ "%s", _("cannot connect to qemu"));
+ close(*qemusock);
+ VIR_FREE(qemusock);
+ goto cleanup;
+ }
+
+ st->driver = &migrateFileDrv;
+ st->privateData = qemusock;
+ ret = 0;
+
+cleanup:
+ if (vm)
+ virDomainObjUnlock(vm);
+
+ qemuDriverUnlock(driver);
+
+ return ret;
+}
+
+/* FIXME: again, if we can use migration exec support with unix domain sockets,
+ * this entire function goes away
+ */
+static int qemu_listen(int port)
+{
+ int qemu_sock, optval;
+ struct sockaddr_in sa_qemu;
+
+ /* NOTE: on error, we just cleanup after ourselves and return -1; the
+ * higher layer will report the error for us
+ */
+
+ qemu_sock = socket(AF_INET, SOCK_STREAM, 0);
+ if (qemu_sock < 0)
+ return -1;
+
+ optval = 1;
+ if (setsockopt(qemu_sock, SOL_SOCKET, SO_REUSEADDR, (char *)&optval,
+ sizeof(optval)) < 0)
+ goto close_qemu_sock;
+
+ memset(&sa_qemu, 0, sizeof(sa_qemu));
+ sa_qemu.sin_port = htons(port);
+ sa_qemu.sin_family = AF_INET;
+ sa_qemu.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+ if (bind(qemu_sock, (struct sockaddr *)&sa_qemu, sizeof(sa_qemu)) < 0)
+ goto close_qemu_sock;
+ if (listen(qemu_sock, 1) < 0)
+ goto close_qemu_sock;
+
+ return qemu_sock;
+
+close_qemu_sock:
+ close(qemu_sock);
+
+ return -1;
+}
+
+static int doSecureMigrate(virDomainPtr dom,
+ virDomainObjPtr vm,
+ const char *uri)
+{
+#define MAX_BUFFER 65536
+ int port;
+ int client_sock, qemu_sock;
+ struct sockaddr_in sa_client;
+ socklen_t addrlen;
+ virConnectPtr dconn;
+ char cmd[HOST_NAME_MAX+50];
+ char *info = NULL;
+ int retval = -1;
+ ssize_t bytes;
+ char buffer[MAX_BUFFER];
+ char *safe_uri;
+ char ebuf[1024];
+ virStreamPtr st;
+
+ /* cycle through the localhost ports 9000 to 10000, looking for a socket
+ * we can listen on. Once we've found one, break out of here
+ */
+ qemu_sock = -1;
+ for (port = 9000; port < 10000; port++) {
+ qemu_sock = qemu_listen(port);
+ if (qemu_sock >= 0)
+ break;
+ }
+
+ if (qemu_sock < 0) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("Could not open secure listening socket: %s"),
+ virStrerror(errno, ebuf, sizeof ebuf));
+ return -1;
+ }
+
+ /* Do the migrate command, but let it return immediately. Then we
+ * will accept data below
+ */
+ snprintf (cmd, sizeof cmd, "migrate -d tcp:127.0.0.1:%d", port);
+
+ if (qemudMonitorCommand(vm, cmd, &info) < 0) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ "%s", _("migrate operation failed"));
+ goto close_qemu_sock;
+ }
+
+ if (strstr(info, "fail") != NULL) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("secure migrate failed: %s"), info);
+ goto close_qemu_sock;
+ }
+
+ addrlen = sizeof(sa_client);
+ while ((client_sock = accept(qemu_sock, (struct sockaddr *)&sa_client, &addrlen)) < 0) {
+ if (errno == EAGAIN || errno == EINTR)
+ continue;
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("Failed accepting from the qemu socket: %s"),
+ virStrerror(errno, ebuf, sizeof ebuf));
+ goto qemu_cancel_migration;
+ }
+
+ safe_uri = qemudEscapeMonitorArg (uri);
+ if (!safe_uri) {
+ virReportOOMError (dom->conn);
+ goto close_client_sock;
+ }
+
+ dconn = virConnectOpen(safe_uri);
+ VIR_FREE (safe_uri);
+ if (dconn == NULL) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("Failed to connect to remote libvirtd"));
+ goto close_client_sock;
+ }
+
+ st = virStreamNew(dconn, 0);
+ if (st == NULL)
+ /* FIXME: do we need to set an error here or did virStreamNew do it? */
+ goto close_dconn;
+
+ if (virDomainMigratePrepareTunnel(dconn, st, vm->def->uuid, 0) < 0)
+ /* FIXME: do we need to set an error here or did PrepareTunnel do it? */
+ goto close_stream;
+
+ for (;;) {
+ bytes = saferead(client_sock, buffer, MAX_BUFFER);
+ if (bytes < 0) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("Failed to read from qemu: %s"),
+ virStrerror(errno, ebuf, sizeof ebuf));
+ goto close_stream;
+ }
+ else if (bytes == 0)
+ /* EOF; get out of here */
+ break;
+
+ if (virStreamSend(st, buffer, bytes) < 0) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("Failed to write migration data to remote libvirtd"));
+ goto close_stream;
+ }
+ }
+
+ virStreamFinish(st);
+ /* FIXME: check for errors */
+
+ retval = 0;
+
+close_stream:
+ virStreamFree(st);
+
+close_dconn:
+ virConnectClose(dconn);
+
+close_client_sock:
+ close(client_sock);
+
+qemu_cancel_migration:
+ if (retval != 0)
+ qemudMonitorCommand(vm, "migrate_cancel", &info);
+ VIR_FREE(info);
+
+close_qemu_sock:
+ close(qemu_sock);
+
+ return retval;
+}
+
/* Perform is the second step, and it runs on the source host. */
static int
qemudDomainMigratePerform (virDomainPtr dom,
const char *cookie ATTRIBUTE_UNUSED,
int cookielen ATTRIBUTE_UNUSED,
const char *uri,
- unsigned long flags ATTRIBUTE_UNUSED,
+ unsigned long flags,
const char *dname ATTRIBUTE_UNUSED,
unsigned long resource)
{
@@ -5128,28 +5437,36 @@ qemudDomainMigratePerform (virDomainPtr dom,
VIR_FREE (info);
}
- /* Issue the migrate command. */
- safe_uri = qemudEscapeMonitorArg (uri);
- if (!safe_uri) {
- virReportOOMError (dom->conn);
- goto cleanup;
- }
- snprintf (cmd, sizeof cmd, "migrate \"%s\"", safe_uri);
- VIR_FREE (safe_uri);
+ if (!(flags & VIR_MIGRATE_TUNNELLED)) {
+ /* Issue the migrate command. */
+ safe_uri = qemudEscapeMonitorArg (uri);
+ if (!safe_uri) {
+ virReportOOMError (dom->conn);
+ goto cleanup;
+ }
+ snprintf (cmd, sizeof cmd, "migrate \"%s\"", safe_uri);
+ VIR_FREE (safe_uri);
- if (qemudMonitorCommand (vm, cmd, &info) < 0) {
- qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
- "%s", _("migrate operation failed"));
- goto cleanup;
- }
+ if (qemudMonitorCommand (vm, cmd, &info) < 0) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ "%s", _("migrate operation failed"));
+ goto cleanup;
+ }
- DEBUG ("%s: migrate reply: %s", vm->def->name, info);
+ DEBUG ("%s: migrate reply: %s", vm->def->name, info);
- /* Now check for "fail" in the output string */
- if (strstr(info, "fail") != NULL) {
- qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
- _("migrate failed: %s"), info);
- goto cleanup;
+ /* Now check for "fail" in the output string */
+ if (strstr(info, "fail") != NULL) {
+ qemudReportError (dom->conn, dom, NULL, VIR_ERR_OPERATION_FAILED,
+ _("migrate failed: %s"), info);
+ goto cleanup;
+ }
+ }
+ else {
+ if (doSecureMigrate(dom, vm, uri) < 0) {
+ /* doSecureMigrate already set the error, so just get out */
+ goto cleanup;
+ }
}
/* Clean up the source domain. */
@@ -5444,6 +5761,7 @@ static virDriver qemuDriver = {
qemudNodeDeviceReset, /* nodeDeviceReset */
NULL,
NULL,
+ qemudDomainMigratePrepareTunnel, /* domainMigratePrepareTunnel */
};
diff --git a/src/remote_internal.c b/src/remote_internal.c
index a078688..d0a2a00 100644
--- a/src/remote_internal.c
+++ b/src/remote_internal.c
@@ -994,7 +994,7 @@ remoteOpen (virConnectPtr conn,
struct private_data *priv;
int ret, rflags = 0;
- if (inside_daemon)
+ if (inside_daemon && (!conn->uri || (conn->uri && !conn->uri->server)))
return VIR_DRV_OPEN_DECLINED;
if (!(priv = remoteAllocPrivateData(conn)))
@@ -6279,7 +6279,7 @@ remoteStreamPacket(virStreamPtr st,
const char *data,
size_t nbytes)
{
- DEBUG("st=%p status=%d data=%p nbytes=%d", st, status, data, nbytes);
+ DEBUG("st=%p status=%d data=%p nbytes=%zu", st, status, data, nbytes);
struct private_data *priv = st->conn->privateData;
struct private_stream_data *privst = st->privateData;
XDR xdr;
@@ -6339,7 +6339,7 @@ remoteStreamPacket(virStreamPtr st,
if (status == REMOTE_CONTINUE) {
if (((4 + REMOTE_MESSAGE_MAX) - thiscall->bufferLength) < nbytes) {
errorf(st->conn,
- VIR_ERR_RPC, _("data size %d too large for payload %d"),
+ VIR_ERR_RPC, _("data size %zu too large for payload %d"),
nbytes, ((4 + REMOTE_MESSAGE_MAX) - thiscall->bufferLength));
goto error;
}
@@ -6398,7 +6398,7 @@ remoteStreamSend(virStreamPtr st,
const char *data,
size_t nbytes)
{
- DEBUG("st=%p data=%p nbytes=%d", st, data, nbytes);
+ DEBUG("st=%p data=%p nbytes=%zu", st, data, nbytes);
struct private_data *priv = st->conn->privateData;
int rv = -1;
@@ -6572,6 +6572,46 @@ done:
return rv;
}
+static int
+remoteDomainMigratePrepareTunnel (virConnectPtr conn,
+ virStreamPtr st,
+ const unsigned char *uuid,
+ unsigned long flags)
+{
+ struct private_data *priv = conn->privateData;
+ struct private_stream_data *privst = NULL;
+ int rv = -1;
+ remote_domain_migrate_prepare_tunnel_args args;
+
+ remoteDriverLock(priv);
+
+ if (!(privst = remoteStreamOpen(st, 1, REMOTE_PROC_DOMAIN_MIGRATE_PREPARE_TUNNEL, priv->counter)))
+ goto done;
+
+ args.uuid = (char *) uuid;
+ args.flags = flags;
+
+ if (call (conn, priv, 0, REMOTE_PROC_DOMAIN_MIGRATE_PREPARE_TUNNEL,
+ (xdrproc_t) xdr_remote_domain_migrate_prepare_tunnel_args, (char *) &args,
+ (xdrproc_t) xdr_void, NULL) == -1) {
+ remoteStreamRelease(priv, privst);
+ goto done;
+ }
+
+ st->driver = &remoteStreamDrv;
+ st->privateData = privst;
+
+ rv = 0;
+
+done:
+ remoteDriverUnlock(priv);
+
+ if (rv == 0 &&
+ st->appSource)
+ rv = virStreamSendAll(st);
+
+ return rv;
+}
/*----------------------------------------------------------------------*/
@@ -7847,6 +7887,7 @@ static virDriver driver = {
remoteNodeDeviceReset, /* nodeDeviceReset */
remoteConnectPutFile,
NULL,
+ remoteDomainMigratePrepareTunnel, /* domainMigratePrepareTunnel */
};
static virNetworkDriver network_driver = {
diff --git a/src/test.c b/src/test.c
index 4480922..945d0ef 100644
--- a/src/test.c
+++ b/src/test.c
@@ -3888,6 +3888,7 @@ static virDriver testDriver = {
NULL, /* nodeDeviceReset */
testConnectPutFile,
testConnectGetFile,
+ NULL, /* domainMigratePrepareTunnel */
};
static virNetworkDriver testNetworkDriver = {
diff --git a/src/uml_driver.c b/src/uml_driver.c
index 53e7e10..b40f8cf 100644
--- a/src/uml_driver.c
+++ b/src/uml_driver.c
@@ -1855,6 +1855,7 @@ static virDriver umlDriver = {
NULL, /* nodeDeviceReset */
NULL,
NULL,
+ NULL, /* domainMigratePrepareTunnel */
};
diff --git a/src/vbox/vbox_tmpl.c b/src/vbox/vbox_tmpl.c
index 5e18fb4..3e440b1 100644
--- a/src/vbox/vbox_tmpl.c
+++ b/src/vbox/vbox_tmpl.c
@@ -4966,6 +4966,7 @@ virDriver NAME(Driver) = {
NULL, /* nodeDeviceReset */
NULL,
NULL,
+ NULL, /* domainMigratePrepareTunnel */
};
virNetworkDriver NAME(NetworkDriver) = {
diff --git a/src/virsh.c b/src/virsh.c
index 9e635d4..cb969e5 100644
--- a/src/virsh.c
+++ b/src/virsh.c
@@ -2414,6 +2414,7 @@ static const vshCmdInfo info_migrate[] = {
static const vshCmdOptDef opts_migrate[] = {
{"live", VSH_OT_BOOL, 0, gettext_noop("live migration")},
+ {"tunnelled", VSH_OT_BOOL, 0, gettext_noop("tunnelled migration")},
{"domain", VSH_OT_DATA, VSH_OFLAG_REQ, gettext_noop("domain name, id or uuid")},
{"desturi", VSH_OT_DATA, VSH_OFLAG_REQ, gettext_noop("connection URI of the destination host")},
{"migrateuri", VSH_OT_DATA, 0, gettext_noop("migration URI, usually can be omitted")},
@@ -2451,6 +2452,9 @@ cmdMigrate (vshControl *ctl, const vshCmd *cmd)
if (vshCommandOptBool (cmd, "live"))
flags |= VIR_MIGRATE_LIVE;
+ if (vshCommandOptBool (cmd, "tunnelled"))
+ flags |= VIR_MIGRATE_TUNNELLED;
+
/* Temporarily connect to the destination host. */
dconn = virConnectOpenAuth (desturi, virConnectAuthPtrDefault, 0);
if (!dconn) goto done;
diff --git a/src/xen_unified.c b/src/xen_unified.c
index b3f8a91..0dddb94 100644
--- a/src/xen_unified.c
+++ b/src/xen_unified.c
@@ -1724,6 +1724,7 @@ static virDriver xenUnifiedDriver = {
xenUnifiedNodeDeviceReset, /* nodeDeviceReset */
NULL,
NULL,
+ NULL, /* domainMigratePrepareTunnel */
};
/**
--
Libvir-list mailing list
Libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list