[PATCH 1/2] qemu: Add qemuProcessSetupPid()

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Setting up cgroups and other things for all kinds of threads (the
emulator thread, vCPU threads, I/O threads) was copy-pasted every time
new thing was added.  Over time each one of those functions changed a
bit differently.  So create one function that does all that setup and
start using it.  That will shave some duplicated code and maybe fix some
bugs as well.

Signed-off-by: Martin Kletzander <mkletzan@xxxxxxxxxx>
---
 src/qemu/qemu_process.c | 278 +++++++++++++++---------------------------------
 1 file changed, 87 insertions(+), 191 deletions(-)

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 215fe5f2f210..d1247d2fd0f9 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -2306,70 +2306,108 @@ qemuProcessSetLinkStates(virQEMUDriverPtr driver,
 }


+/**
+ * qemuProcessSetupPid:
+ *
+ * This function sets resource properities (affinity, cgroups,
+ * scheduler) for any PID associated with a domain.  It should be used
+ * to set up emulator PIDs as well as vCPU and I/O thread pids to
+ * ensure they are all handled the same way.
+ *
+ * Returns 0 on success, -1 on error.
+ */
 static int
-qemuProcessSetupEmulator(virDomainObjPtr vm)
+qemuProcessSetupPid(virDomainObjPtr vm,
+                    pid_t pid,
+                    virCgroupThreadName nameval,
+                    int id,
+                    virBitmapPtr cpumask,
+                    int sched_policy,
+                    int sched_priority)
 {
-    virBitmapPtr cpumask = NULL;
-    virCgroupPtr cgroup_emulator = NULL;
     qemuDomainObjPrivatePtr priv = vm->privateData;
-    unsigned long long period = vm->def->cputune.emulator_period;
-    long long quota = vm->def->cputune.emulator_quota;
+    unsigned long long period = vm->def->cputune.period;
+    long long quota = vm->def->cputune.quota;
+    virDomainNumatuneMemMode mem_mode;
+    virCgroupPtr cgroup = NULL;
+    virBitmapPtr use_cpumask;
+    char *mem_mask = NULL;
     int ret = -1;

     if ((period || quota) &&
         !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                        _("cgroup cpu is required for scheduler tuning"));
-        return -1;
+        goto cleanup;
     }

-    if (vm->def->cputune.emulatorpin)
-        cpumask = vm->def->cputune.emulatorpin;
-    else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO &&
-             priv->autoCpuset)
-        cpumask = priv->autoCpuset;
+    /* Infer which cpumask shall be used. */
+    if (cpumask)
+        use_cpumask = cpumask;
+    else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
+        use_cpumask = priv->autoCpuset;
     else
-        cpumask = vm->def->cpumask;
+        use_cpumask = vm->def->cpumask;

-    /* If CPU cgroup controller is not initialized here, then we need
+    /*
+     * If CPU cgroup controller is not initialized here, then we need
      * neither period nor quota settings.  And if CPUSET controller is
-     * not initialized either, then there's nothing to do anyway. */
+     * not initialized either, then there's nothing to do anyway.
+     */
     if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) ||
         virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {

-        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_EMULATOR, 0,
-                               true, &cgroup_emulator) < 0)
+        if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
+            mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
+            virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
+                                                priv->autoNodeset,
+                                                &mem_mask, -1) < 0)
             goto cleanup;

-        if (virCgroupAddTask(cgroup_emulator, vm->pid) < 0)
+        if (virCgroupNewThread(priv->cgroup, nameval, id, true, &cgroup) < 0)
             goto cleanup;

+        /* move the thread for vcpu to sub dir */
+        if (virCgroupAddTask(cgroup, pid) < 0)
+            goto cleanup;

-        if (cpumask) {
-            if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET) &&
-                qemuSetupCgroupCpusetCpus(cgroup_emulator, cpumask) < 0)
+        if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
+            if (use_cpumask &&
+                qemuSetupCgroupCpusetCpus(cgroup, use_cpumask) < 0)
                 goto cleanup;
-        }

-        if (period || quota) {
-            if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) &&
-                qemuSetupCgroupVcpuBW(cgroup_emulator, period,
-                                      quota) < 0)
+            /*
+             * Don't setup cpuset.mems for the emulator, they need to
+             * be set up after initialization in order for kvm
+             * allocations to succeed.
+             */
+            if (nameval != VIR_CGROUP_THREAD_EMULATOR &&
+                mem_mask && virCgroupSetCpusetMems(cgroup, mem_mask) < 0)
                 goto cleanup;
+
         }
+
+        if ((period || quota) &&
+            qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0)
+            goto cleanup;
     }

-    if (cpumask &&
-        virProcessSetAffinity(vm->pid, cpumask) < 0)
+    /* Setup legacy affinity. */
+    if (use_cpumask && virProcessSetAffinity(pid, use_cpumask) < 0)
         goto cleanup;

-    ret = 0;
+    /* Set scheduler type and priority. */
+    if (sched_policy != VIR_PROC_POLICY_NONE &&
+        virProcessSetScheduler(pid, sched_policy, sched_priority) < 0)
+        goto cleanup;

+    ret = 0;
  cleanup:
-    if (cgroup_emulator) {
+    VIR_FREE(mem_mask);
+    if (cgroup) {
         if (ret < 0)
-            virCgroupRemove(cgroup_emulator);
-        virCgroupFree(&cgroup_emulator);
+            virCgroupRemove(cgroup);
+        virCgroupFree(&cgroup);
     }

     return ret;
@@ -2377,6 +2415,15 @@ qemuProcessSetupEmulator(virDomainObjPtr vm)


 static int
+qemuProcessSetupEmulator(virDomainObjPtr vm)
+{
+    return qemuProcessSetupPid(vm, vm->pid, VIR_CGROUP_THREAD_EMULATOR,
+                               0, vm->def->cputune.emulatorpin,
+                               VIR_PROC_POLICY_NONE, 0);
+}
+
+
+static int
 qemuProcessInitPasswords(virConnectPtr conn,
                          virQEMUDriverPtr driver,
                          virDomainObjPtr vm,
@@ -4574,80 +4621,10 @@ qemuProcessSetupVcpu(virDomainObjPtr vm,
 {
     pid_t vcpupid = qemuDomainGetVcpuPid(vm, vcpuid);
     virDomainVcpuInfoPtr vcpu = virDomainDefGetVcpu(vm->def, vcpuid);
-    qemuDomainObjPrivatePtr priv = vm->privateData;
-    char *mem_mask = NULL;
-    virDomainNumatuneMemMode mem_mode;
-    unsigned long long period = vm->def->cputune.period;
-    long long quota = vm->def->cputune.quota;
-    virCgroupPtr cgroup_vcpu = NULL;
-    virBitmapPtr cpumask;
-    int ret = -1;
-
-    if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) ||
-        virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-
-        if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
-            mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
-            virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
-                                                priv->autoNodeset,
-                                                &mem_mask, -1) < 0)
-            goto cleanup;
-
-        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_VCPU, vcpuid,
-                               true, &cgroup_vcpu) < 0)
-            goto cleanup;
-
-        if (period || quota) {
-            if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
-                goto cleanup;
-        }
-    }
-
-    /* infer which cpumask shall be used */
-    if (vcpu->cpumask)
-        cpumask = vcpu->cpumask;
-    else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
-        cpumask = priv->autoCpuset;
-    else
-        cpumask = vm->def->cpumask;
-
-    /* setup cgroups */
-    if (cgroup_vcpu) {
-        if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-            if (mem_mask && virCgroupSetCpusetMems(cgroup_vcpu, mem_mask) < 0)
-                goto cleanup;
-
-            if (cpumask && qemuSetupCgroupCpusetCpus(cgroup_vcpu, cpumask) < 0)
-                goto cleanup;
-        }
-
-        /* move the thread for vcpu to sub dir */
-        if (virCgroupAddTask(cgroup_vcpu, vcpupid) < 0)
-            goto cleanup;
-    }
-
-    /* setup legacy affinty */
-    if (cpumask && virProcessSetAffinity(vcpupid, cpumask) < 0)
-        goto cleanup;
-
-    /* set scheduler type and priority */
-    if (vcpu->sched.policy != VIR_PROC_POLICY_NONE) {
-        if (virProcessSetScheduler(vcpupid, vcpu->sched.policy,
-                                   vcpu->sched.priority) < 0)
-            goto cleanup;
-    }
-
-    ret = 0;
-
- cleanup:
-    VIR_FREE(mem_mask);
-    if (cgroup_vcpu) {
-        if (ret < 0)
-            virCgroupRemove(cgroup_vcpu);
-        virCgroupFree(&cgroup_vcpu);
-    }

-    return ret;
+    return qemuProcessSetupPid(vm, vcpupid, VIR_CGROUP_THREAD_VCPU,
+                               vcpuid, vcpu->cpumask,
+                               vcpu->sched.policy, vcpu->sched.priority);
 }


@@ -4700,98 +4677,17 @@ qemuProcessSetupVcpus(virDomainObjPtr vm)
 }


-/**
- * qemuProcessSetupIOThread:
- * @vm: domain object
- * @iothread: iothread data structure to set the data for
- *
- * This function sets resource properities (affinity, cgroups, scheduler) for a
- * IOThread. This function expects that the IOThread is online and the IOThread
- * pids were correctly detected at the point when it's called.
- *
- * Returns 0 on success, -1 on error.
- */
 int
 qemuProcessSetupIOThread(virDomainObjPtr vm,
                          virDomainIOThreadIDDefPtr iothread)
 {
-    qemuDomainObjPrivatePtr priv = vm->privateData;
-    unsigned long long period = vm->def->cputune.period;
-    long long quota = vm->def->cputune.quota;
-    virDomainNumatuneMemMode mem_mode;
-    char *mem_mask = NULL;
-    virCgroupPtr cgroup_iothread = NULL;
-    virBitmapPtr cpumask = NULL;
-    int ret = -1;
-
-    if ((period || quota) &&
-        !virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU)) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("cgroup cpu is required for scheduler tuning"));
-        return -1;
-    }
-
-    if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPU) ||
-        virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-        if (virDomainNumatuneGetMode(vm->def->numa, -1, &mem_mode) == 0 &&
-            mem_mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
-            virDomainNumatuneMaybeFormatNodeset(vm->def->numa,
-                                                priv->autoNodeset,
-                                                &mem_mask, -1) < 0)
-            goto cleanup;

-        if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_IOTHREAD,
+    return qemuProcessSetupPid(vm, iothread->thread_id,
+                               VIR_CGROUP_THREAD_IOTHREAD,
                                iothread->iothread_id,
-                               true, &cgroup_iothread) < 0)
-            goto cleanup;
-    }
-
-    if (iothread->cpumask)
-        cpumask = iothread->cpumask;
-    else if (vm->def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO)
-        cpumask = priv->autoCpuset;
-    else
-        cpumask = vm->def->cpumask;
-
-    if (period || quota) {
-        if (qemuSetupCgroupVcpuBW(cgroup_iothread, period, quota) < 0)
-            goto cleanup;
-    }
-
-    if (cgroup_iothread) {
-        if (virCgroupHasController(priv->cgroup, VIR_CGROUP_CONTROLLER_CPUSET)) {
-            if (mem_mask &&
-                virCgroupSetCpusetMems(cgroup_iothread, mem_mask) < 0)
-                goto cleanup;
-
-            if (cpumask &&
-                qemuSetupCgroupCpusetCpus(cgroup_iothread, cpumask) < 0)
-                goto cleanup;
-        }
-
-        if (virCgroupAddTask(cgroup_iothread, iothread->thread_id) < 0)
-            goto cleanup;
-    }
-
-    if (cpumask && virProcessSetAffinity(iothread->thread_id, cpumask) < 0)
-        goto cleanup;
-
-    if (iothread->sched.policy != VIR_PROC_POLICY_NONE &&
-        virProcessSetScheduler(iothread->thread_id, iothread->sched.policy,
-                               iothread->sched.priority) < 0)
-        goto cleanup;
-
-    ret = 0;
-
- cleanup:
-    if (cgroup_iothread) {
-        if (ret < 0)
-            virCgroupRemove(cgroup_iothread);
-        virCgroupFree(&cgroup_iothread);
-    }
-
-    VIR_FREE(mem_mask);
-    return ret;
+                               iothread->cpumask,
+                               iothread->sched.policy,
+                               iothread->sched.priority);
 }


@@ -5292,7 +5188,7 @@ qemuProcessLaunch(virConnectPtr conn,
     if (!qemuProcessVerifyGuestCPU(driver, vm, asyncJob))
         goto cleanup;

-    VIR_DEBUG("Setting up post-init cgroup restrictions");
+    VIR_DEBUG("Setting Post-init cgroup resctrictions");
     if (qemuSetupCpusetMems(vm) < 0)
         goto cleanup;

-- 
2.9.0

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list



[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]