[PATCH V2 16/20] qemu: Support O_DIRECT with mapped-ram on restore

[Jim Fehlig via Devel <devel@xxxxxxxxxxxxxxxxx>]

When using the mapped-ram migration capability, direct IO is
enabled by setting the "direct-io" migration parameter to
"true" and passing QEMU an additional fd with O_DIRECT set.

Signed-off-by: Jim Fehlig <jfehlig@xxxxxxxx>
---
 src/qemu/qemu_migration.c | 11 ++++++-----
 src/qemu/qemu_process.c   | 27 +++++++++++++++++++++++----
 src/qemu/qemu_process.h   |  6 ++++--
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
index 61a14dc9d6..84d7710462 100644
--- a/src/qemu/qemu_migration.c
+++ b/src/qemu/qemu_migration.c
@@ -3059,7 +3059,8 @@ qemuMigrationDstPrepareCleanup(virQEMUDriver *driver,
 }
 
 static qemuProcessIncomingDef *
-qemuMigrationDstPrepare(virDomainObj *vm,
+qemuMigrationDstPrepare(virQEMUDriver *driver,
+                        virDomainObj *vm,
                         bool tunnel,
                         const char *protocol,
                         const char *listenAddress,
@@ -3119,9 +3120,9 @@ qemuMigrationDstPrepare(virDomainObj *vm,
         migrateFrom = g_strdup_printf(incFormat, protocol, listenAddress, port);
     }
 
-    return qemuProcessIncomingDefNew(vm, listenAddress,
+    return qemuProcessIncomingDefNew(driver, vm, listenAddress,
                                      migrateFrom, fd,
-                                     NULL, NULL);
+                                     NULL, NULL, NULL);
 }
 
 
@@ -3261,7 +3262,7 @@ qemuMigrationDstPrepareActive(virQEMUDriver *driver,
         goto error;
     stopProcess = true;
 
-    if (!(incoming = qemuMigrationDstPrepare(vm, tunnel, protocol,
+    if (!(incoming = qemuMigrationDstPrepare(driver, vm, tunnel, protocol,
                                              listenAddress, port,
                                              &dataFD[0])))
         goto error;
@@ -3633,7 +3634,7 @@ qemuMigrationDstPrepareResume(virQEMUDriver *driver,
 
     priv->origname = g_strdup(origname);
 
-    if (!(incoming = qemuMigrationDstPrepare(vm, false, protocol,
+    if (!(incoming = qemuMigrationDstPrepare(driver, vm, false, protocol,
                                              listenAddress, port, NULL)))
         goto cleanup;
 
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index e5ad6f0528..ad926650f7 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -4818,13 +4818,16 @@ qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc)
  * qemuProcessIncomingDefFree will NOT close it.
  */
 qemuProcessIncomingDef *
-qemuProcessIncomingDefNew(virDomainObj *vm,
+qemuProcessIncomingDefNew(virQEMUDriver *driver,
+                          virDomainObj *vm,
                           const char *listenAddress,
                           const char *migrateFrom,
                           int *fd,
                           const char *path,
-                          virQEMUSaveData *data)
+                          virQEMUSaveData *data,
+                          qemuMigrationParams *migParams)
 {
+    g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
     qemuDomainObjPrivate *priv = vm->privateData;
     qemuProcessIncomingDef *inc = NULL;
 
@@ -4839,10 +4842,26 @@ qemuProcessIncomingDefNew(virDomainObj *vm,
         size_t offset = sizeof(virQEMUSaveHeader) + data->header.data_len;
 
         if (data->header.format == QEMU_SAVE_FORMAT_SPARSE) {
+            bool directio = false;
             unsigned int fdsetId;
 
             inc->fdPassMigrate = qemuFDPassNew("migrate", priv);
-            qemuFDPassAddFD(inc->fdPassMigrate, fd, "-fd");
+            /* When using directio with mapped-ram, qemu needs an fd without
+             * O_DIRECT set for reading small bits of unaligned state. */
+            if (qemuMigrationParamsGetBool(migParams, QEMU_MIGRATION_PARAM_DIRECT_IO, &directio) < 0)
+                goto error;
+
+            if (directio) {
+                VIR_AUTOCLOSE bufferedFd = -1;
+
+                if ((bufferedFd = qemuDomainOpenFile(cfg, NULL, path, O_RDONLY, NULL)) < 0)
+                    goto error;
+
+                qemuFDPassAddFD(inc->fdPassMigrate, &bufferedFd, "-buffered-fd");
+                qemuFDPassAddFD(inc->fdPassMigrate, fd, "direct-io-fd");
+            } else {
+                qemuFDPassAddFD(inc->fdPassMigrate, fd, "-buffered-fd");
+            }
             qemuFDPassGetId(inc->fdPassMigrate, &fdsetId);
             inc->uri = g_strdup_printf("file:/dev/fdset/%u,offset=%#lx", fdsetId, offset);
         } else {
@@ -8481,7 +8500,7 @@ qemuProcessStartWithMemoryState(virConnectPtr conn,
     int rc = 0;
     int ret = -1;
 
-    incoming = qemuProcessIncomingDefNew(vm, NULL, "stdio", fd, path, data);
+    incoming = qemuProcessIncomingDefNew(driver, vm, NULL, "stdio", fd, path, data, migParams);
     if (!incoming)
         return -1;
 
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index a6258b075c..3c70835ac1 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -57,12 +57,14 @@ struct _qemuProcessIncomingDef {
     const char *path; /* path associated with fd */
 };
 
-qemuProcessIncomingDef *qemuProcessIncomingDefNew(virDomainObj *vm,
+qemuProcessIncomingDef *qemuProcessIncomingDefNew(virQEMUDriver *driver,
+                                                  virDomainObj *vm,
                                                   const char *listenAddress,
                                                   const char *migrateFrom,
                                                   int *fd,
                                                   const char *path,
-                                                  virQEMUSaveData *data);
+                                                  virQEMUSaveData *data,
+                                                  qemuMigrationParams *migParams);
 
 void qemuProcessIncomingDefFree(qemuProcessIncomingDef *inc);
 
-- 
2.43.0