[PATCH V3 15/19] qemu: Support O_DIRECT with mapped-ram on restore

[Jim Fehlig via Devel <devel@xxxxxxxxxxxxxxxxx>]

When using the mapped-ram migration capability, direct IO is
enabled by setting the "direct-io" migration parameter to
"true" and passing QEMU an additional fd with O_DIRECT set.

Signed-off-by: Jim Fehlig <jfehlig@xxxxxxxx>
---
 src/qemu/qemu_migration.c | 11 ++++++-----
 src/qemu/qemu_process.c   | 27 +++++++++++++++++++++++----
 src/qemu/qemu_process.h   |  6 ++++--
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index a5a80230b2..d0cbcce044 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -3063,7 +3063,8 @@ qemuMigrationDstPrepareCleanup(virQEMUDriver *driver,
 }
 
 static qemuProcessIncomingDef *
-qemuMigrationDstPrepare(virDomainObj *vm,
+qemuMigrationDstPrepare(virQEMUDriver *driver,
+                        virDomainObj *vm,
                         bool tunnel,
                         const char *protocol,
                         const char *listenAddress,
@@ -3123,9 +3124,9 @@ qemuMigrationDstPrepare(virDomainObj *vm,
         migrateFrom = g_strdup_printf(incFormat, protocol, listenAddress, port);
     }
 
-    return qemuProcessIncomingDefNew(vm, listenAddress,
+    return qemuProcessIncomingDefNew(driver, vm, listenAddress,
                                      migrateFrom, fd,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 }
 
 
@@ -3265,7 +3266,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver,
         goto error;
     stopProcess = true;
 
-    if (!(incoming = qemuMigrationDstPrepare(vm, tunnel, protocol,
+    if (!(incoming = qemuMigrationDstPrepare(driver, vm, tunnel, protocol,
                                              listenAddress, port,
                                              &dataFD[0])))
         goto error;
@@ -3637,7 +3638,7 @@ qemuMigrationDstPrepareResume(virQEMUDriver *driver,
 
     priv->origname = g_strdup(origname);
 
-    if (!(incoming = qemuMigrationDstPrepare(vm, false, protocol,
+    if (!(incoming = qemuMigrationDstPrepare(driver, vm, false, protocol,
                                              listenAddress, port, NULL)))
         goto cleanup;
 
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 94e7e90d28..f51378aeba 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -4837,13 +4837,16 @@ qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc)
  * qemuProcessIncomingDefFree will NOT close it.
  */
 qemuProcessIncomingDef *
-qemuProcessIncomingDefNew(virDomainObj *vm,
+qemuProcessIncomingDefNew(virQEMUDriver *driver,
+                          virDomainObj *vm,
                           const char *listenAddress,
                           const char *migrateFrom,
                           int *fd,
                           const char *path,
-                          virQEMUSaveData *data)
+                          virQEMUSaveData *data,
+                          qemuMigrationParams *migParams)
 {
+    g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
     qemuDomainObjPrivate *priv = vm->privateData;
     qemuProcessIncomingDef *inc = NULL;
 
@@ -4856,9 +4859,25 @@ qemuProcessIncomingDefNew(virDomainObj *vm,
 
     if (data && data->header.format == QEMU_SAVE_FORMAT_SPARSE) {
         size_t offset = sizeof(virQEMUSaveHeader) + data->header.data_len;
+        bool directio = false;
 
         inc->fdPassMigrate = qemuFDPassNew("libvirt-incoming-migrate", priv);
-        qemuFDPassAddFD(inc->fdPassMigrate, fd, "-fd");
+        /* When using directio with mapped-ram, qemu needs an fd without
+         * O_DIRECT set for reading small bits of unaligned state. */
+        if (qemuMigrationParamsGetBool(migParams, QEMU_MIGRATION_PARAM_DIRECT_IO, &directio) < 0)
+            goto error;
+
+        if (directio) {
+            VIR_AUTOCLOSE bufferedFd = -1;
+
+            if ((bufferedFd = qemuDomainOpenFile(cfg, NULL, path, O_RDONLY, NULL)) < 0)
+                goto error;
+
+            qemuFDPassAddFD(inc->fdPassMigrate, &bufferedFd, "-buffered-fd");
+            qemuFDPassAddFD(inc->fdPassMigrate, fd, "direct-io-fd");
+        } else {
+            qemuFDPassAddFD(inc->fdPassMigrate, fd, "-buffered-fd");
+        }
         inc->uri = g_strdup_printf("file:%s,offset=%#lx",
                                    qemuFDPassGetPath(inc->fdPassMigrate), offset);
     } else {
@@ -8499,7 +8518,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn,
     /* The fd passed to qemuProcessIncomingDefNew is used to create the migration
      * URI, so it must be called after starting the decompression program.
      */
-    incoming = qemuProcessIncomingDefNew(vm, NULL, "stdio", fd, path, data);
+    incoming = qemuProcessIncomingDefNew(driver, vm, NULL, "stdio", fd, path, data, migParams);
     if (!incoming)
         return -1;
 
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index c51335ad7a..8f7b3f24c4 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -57,12 +57,14 @@ struct _qemuProcessIncomingDef {
     const char *path; /* path associated with fd */
 };
 
-qemuProcessIncomingDef *qemuProcessIncomingDefNew(virDomainObj *vm,
+qemuProcessIncomingDef *qemuProcessIncomingDefNew(virQEMUDriver *driver,
+                                                  virDomainObj *vm,
                                                   const char *listenAddress,
                                                   const char *migrateFrom,
                                                   int *fd,
                                                   const char *path,
-                                                  virQEMUSaveData *data);
+                                                  virQEMUSaveData *data,
+                                                  qemuMigrationParams *migParams);
 
 void qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc);
 
-- 
2.43.0