[PATCH v2 37/39] qemu: Allow forcing VFIO when computing memlock limit

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



With NVMe disks, one can start a blockjob with a NVMe disk
that is not visible in domain XML (at least right away). Usually,
it's fairly easy to override this limitation of
qemuDomainGetMemLockLimitBytes() - for instance for hostdevs we
temporarily add the device to domain def, let the function
calculate the limit and then remove the device. But it's not so
easy with virStorageSourcePtr - in some cases they don't
necessarily are attached to a disk. And even if they are it's
done later in the process and frankly, I find it too complicated
to be able to use the simple trick we use with hostdevs.

Signed-off-by: Michal Privoznik <mprivozn@xxxxxxxxxx>
---
 src/qemu/qemu_command.c |  2 +-
 src/qemu/qemu_domain.c  | 46 ++++++++++++++++++++++++++---------------
 src/qemu/qemu_domain.h  |  6 ++++--
 src/qemu/qemu_hotplug.c | 12 +++++------
 tests/qemumemlocktest.c |  2 +-
 5 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index ad6c1e7e8a..fa2394378a 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -10511,7 +10511,7 @@ qemuBuildCommandLine(virQEMUDriverPtr driver,
 
     /* In some situations, eg. VFIO passthrough, QEMU might need to lock a
      * significant amount of memory, so we need to set the limit accordingly */
-    virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def));
+    virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def, false));
 
     if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MSG_TIMESTAMP) &&
         cfg->logTimestamp)
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index e4e9d4e361..3fe0004cab 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -11843,12 +11843,14 @@ ppc64VFIODeviceIsNV2Bridge(const char *device)
 /**
  * getPPC64MemLockLimitBytes:
  * @def: domain definition
+ * @forceVFIO: force VFIO usage
  *
  * A PPC64 helper that calculates the memory locking limit in order for
  * the guest to operate properly.
  */
 static unsigned long long
-getPPC64MemLockLimitBytes(virDomainDefPtr def)
+getPPC64MemLockLimitBytes(virDomainDefPtr def,
+                          bool forceVFIO)
 {
     unsigned long long memKB = 0;
     unsigned long long baseLimit = 0;
@@ -11939,7 +11941,7 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
         passthroughLimit = maxMemory +
                            128 * (1ULL<<30) / 512 * nPCIHostBridges +
                            8192;
-    } else if (usesVFIO) {
+    } else if (usesVFIO || forceVFIO) {
         /* For regular (non-NVLink2 present) VFIO passthrough, the value
          * of passthroughLimit is:
          *
@@ -11977,16 +11979,20 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
 /**
  * qemuDomainGetMemLockLimitBytes:
  * @def: domain definition
+ * @forceVFIO: force VFIO calculation
  *
  * Calculate the memory locking limit that needs to be set in order for
  * the guest to operate properly. The limit depends on a number of factors,
  * including certain configuration options and less immediately apparent ones
  * such as the guest architecture or the use of certain devices.
+ * The @forceVFIO argument can be used to tell this function will use VFIO even
+ * though @def doesn't indicates so right now.
  *
  * Returns: the memory locking limit, or 0 if setting the limit is not needed
  */
 unsigned long long
-qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
+qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
+                               bool forceVFIO)
 {
     unsigned long long memKB = 0;
     bool usesVFIO = false;
@@ -12007,7 +12013,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
         return VIR_DOMAIN_MEMORY_PARAM_UNLIMITED;
 
     if (ARCH_IS_PPC64(def->os.arch) && def->virtType == VIR_DOMAIN_VIRT_KVM)
-        return getPPC64MemLockLimitBytes(def);
+        return getPPC64MemLockLimitBytes(def, forceVFIO);
 
     /* For device passthrough using VFIO the guest memory and MMIO memory
      * regions need to be locked persistent in order to allow DMA.
@@ -12027,18 +12033,20 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
      *
      * Note that this may not be valid for all platforms.
      */
-    for (i = 0; i < def->nhostdevs; i++) {
-        if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
-            virHostdevIsMdevDevice(def->hostdevs[i])) {
-            usesVFIO = true;
-            break;
+    if (!forceVFIO) {
+        for (i = 0; i < def->nhostdevs; i++) {
+            if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
+                virHostdevIsMdevDevice(def->hostdevs[i])) {
+                usesVFIO = true;
+                break;
+            }
         }
+
+        if (virDomainDefHasNVMeDisk(def))
+            usesVFIO = true;
     }
 
-    if (virDomainDefHasNVMeDisk(def))
-        usesVFIO = true;
-
-    if (usesVFIO)
+    if (usesVFIO || forceVFIO)
         memKB = virDomainDefGetMemoryTotal(def) + 1024 * 1024;
 
  done:
@@ -12049,9 +12057,12 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
 /**
  * qemuDomainAdjustMaxMemLock:
  * @vm: domain
+ * @forceVFIO: apply VFIO requirements even if vm's def doesn't require it
  *
  * Adjust the memory locking limit for the QEMU process associated to @vm, in
- * order to comply with VFIO or architecture requirements.
+ * order to comply with VFIO or architecture requirements. If @forceVFIO is
+ * true then the limit is changed even if nothing in @vm's definition indicates
+ * so.
  *
  * The limit will not be changed unless doing so is needed; the first time
  * the limit is changed, the original (default) limit is stored in @vm and
@@ -12061,12 +12072,13 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
  * Returns: 0 on success, <0 on failure
  */
 int
-qemuDomainAdjustMaxMemLock(virDomainObjPtr vm)
+qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
+                           bool forceVFIO)
 {
     unsigned long long bytes = 0;
     int ret = -1;
 
-    bytes = qemuDomainGetMemLockLimitBytes(vm->def);
+    bytes = qemuDomainGetMemLockLimitBytes(vm->def, forceVFIO);
 
     if (bytes) {
         /* If this is the first time adjusting the limit, save the current
@@ -12115,7 +12127,7 @@ qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
     int ret = 0;
 
     vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
-    if (qemuDomainAdjustMaxMemLock(vm) < 0)
+    if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
         ret = -1;
 
     vm->def->hostdevs[--(vm->def->nhostdevs)] = NULL;
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index de6479f8e0..7802a8b98b 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -962,8 +962,10 @@ bool qemuDomainSupportsPCI(virDomainDefPtr def,
 
 void qemuDomainUpdateCurrentMemorySize(virDomainObjPtr vm);
 
-unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def);
-int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm);
+unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
+                                                  bool forceVFIO);
+int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
+                               bool forceVFIO);
 int qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
                                       virDomainHostdevDefPtr hostdev);
 
diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index 75ffc099e1..f5a672a195 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -1648,7 +1648,7 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
     if (teardowndevice &&
         qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0)
         VIR_WARN("Unable to remove host device from /dev");
-    if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
+    if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
         VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
 
     if (releaseaddr)
@@ -2387,7 +2387,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
     if (virDomainMemoryInsert(vm->def, mem) < 0)
         goto cleanup;
 
-    if (qemuDomainAdjustMaxMemLock(vm) < 0)
+    if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
         goto removedef;
 
     qemuDomainObjEnterMonitor(driver, vm);
@@ -2458,7 +2458,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
 
     /* reset the mlock limit */
     virErrorPreserveLast(&orig_err);
-    ignore_value(qemuDomainAdjustMaxMemLock(vm));
+    ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
     virErrorRestore(&orig_err);
 
     goto audit;
@@ -2856,7 +2856,7 @@ qemuDomainAttachMediatedDevice(virQEMUDriverPtr driver,
     ret = 0;
  cleanup:
     if (ret < 0) {
-        if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
+        if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
             VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
         if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
             VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
@@ -4378,7 +4378,7 @@ qemuDomainRemoveMemoryDevice(virQEMUDriverPtr driver,
     ignore_value(qemuProcessRefreshBalloonState(driver, vm, QEMU_ASYNC_JOB_NONE));
 
     /* decrease the mlock limit after memory unplug if necessary */
-    ignore_value(qemuDomainAdjustMaxMemLock(vm));
+    ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
 
     return 0;
 }
@@ -4505,7 +4505,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver,
         qemuDomainRemovePCIHostDevice(driver, vm, hostdev);
         /* QEMU might no longer need to lock as much memory, eg. we just
          * detached the last VFIO device, so adjust the limit here */
-        if (qemuDomainAdjustMaxMemLock(vm) < 0)
+        if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
             VIR_WARN("Failed to adjust locked memory limit");
         break;
     case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
diff --git a/tests/qemumemlocktest.c b/tests/qemumemlocktest.c
index c9484ac9cb..1d52498e25 100644
--- a/tests/qemumemlocktest.c
+++ b/tests/qemumemlocktest.c
@@ -43,7 +43,7 @@ testCompareMemLock(const void *data)
         goto cleanup;
     }
 
-    ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def));
+    ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def, false));
 
  cleanup:
     virDomainDefFree(def);
-- 
2.21.0

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list



[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]

  Powered by Linux