[PATCH 04/13] Enable cpuset cgroup and synchronous vcpupin info to cgroup

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



This patch enables cpuset cgroup, and synchronous vcpupin info set by sched_setaffinity() to cgroup.

Signed-off-by: Tang Chen <tangchen@xxxxxxxxxxxxxx>
---
 src/libvirt_private.syms |    2 +
 src/qemu/qemu_cgroup.c   |   51 ++++++++++++++++++++++++++++++++++++++++++++-
 src/qemu/qemu_cgroup.h   |    2 +
 src/qemu/qemu_driver.c   |   43 +++++++++++++++++++++++++++++++-------
 src/util/cgroup.c        |   35 ++++++++++++++++++++++++++++++-
 src/util/cgroup.h        |    3 ++
 6 files changed, 125 insertions(+), 11 deletions(-)

diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 6ff1a3b..88cc37a 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -77,6 +77,7 @@ virCgroupGetCpuShares;
 virCgroupGetCpuacctPercpuUsage;
 virCgroupGetCpuacctStat;
 virCgroupGetCpuacctUsage;
+virCgroupGetCpusetCpus;
 virCgroupGetCpusetMems;
 virCgroupGetFreezerState;
 virCgroupGetMemSwapHardLimit;
@@ -95,6 +96,7 @@ virCgroupSetBlkioWeight;
 virCgroupSetCpuCfsPeriod;
 virCgroupSetCpuCfsQuota;
 virCgroupSetCpuShares;
+virCgroupSetCpusetCpus;
 virCgroupSetCpusetMems;
 virCgroupSetFreezerState;
 virCgroupSetMemSwapHardLimit;
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index e69ef5b..1085478 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -473,18 +473,57 @@ cleanup:
         rc = virCgroupSetCpuCfsPeriod(cgroup, old_period);
         if (rc < 0)
             virReportSystemError(-rc,
-                                 _("%s"),
-                                 "Unable to rollback cpu bandwidth period");
+                                 "%s",
+                                 _("Unable to rollback cpu bandwidth period"));
     }
 
     return -1;
 }
 
+int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
+                           int vcpuid)
+{
+    int i, rc;
+    char *new_cpus = NULL;
+
+    if (vcpuid < 0 || vcpuid >= def->vcpus) {
+        virReportSystemError(EINVAL,
+                             "%s: %d", _("invalid vcpuid"), vcpuid);
+        return -1;
+    }
+
+    for (i = 0; i < def->cputune.nvcpupin; i++) {
+        if (vcpuid == def->cputune.vcpupin[i]->vcpuid) {
+            new_cpus = virDomainCpuSetFormat(def->cputune.vcpupin[i]->cpumask,
+                                             VIR_DOMAIN_CPUMASK_LEN);
+            if (!new_cpus) {
+                qemuReportError(VIR_ERR_INTERNAL_ERROR,
+                                _("failed to convert cpu mask"));
+                goto cleanup;
+            }
+            rc = virCgroupSetCpusetCpus(cgroup, new_cpus);
+            if (rc < 0) {
+                virReportSystemError(-rc,
+                                     "%s", _("Unable to set cpuset.cpus"));
+                goto cleanup;
+            }
+        }
+    }
+    VIR_FREE(new_cpus);
+    return 0;
+
+cleanup:
+    if (new_cpus)
+        VIR_FREE(new_cpus);
+    return -1;
+}
+
 int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
 {
     virCgroupPtr cgroup = NULL;
     virCgroupPtr cgroup_vcpu = NULL;
     qemuDomainObjPrivatePtr priv = vm->privateData;
+    virDomainDefPtr def = vm->def;
     int rc;
     unsigned int i;
     unsigned long long period = vm->def->cputune.period;
@@ -556,6 +595,14 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
             }
         }
 
+        /* Set vcpupin in cgroup if vcpupin xml is provided */
+        if (def->cputune.nvcpupin) {
+            if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
+                if (qemuSetupCgroupVcpuPin(cgroup_vcpu, def, i) < 0)
+                    goto cleanup;
+            }
+        }
+
         virCgroupFree(&cgroup_vcpu);
     }
 
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index cf0d383..91d5632 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -53,6 +53,8 @@ int qemuSetupCgroup(struct qemud_driver *driver,
 int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
                           unsigned long long period,
                           long long quota);
+int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
+                           int vcpuid);
 int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm);
 int qemuSetupCgroupForHypervisor(struct qemud_driver *driver,
                                  virDomainObjPtr vm);
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index d3f74d2..b0eef80 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -3551,6 +3551,8 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
     struct qemud_driver *driver = dom->conn->privateData;
     virDomainObjPtr vm;
     virDomainDefPtr persistentDef = NULL;
+    virCgroupPtr cgroup_dom = NULL;
+    virCgroupPtr cgroup_vcpu = NULL;
     int maxcpu, hostcpus;
     virNodeInfo nodeinfo;
     int ret = -1;
@@ -3605,9 +3607,37 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
     if (flags & VIR_DOMAIN_AFFECT_LIVE) {
 
         if (priv->vcpupids != NULL) {
+            /* Add config to vm->def first, because cgroup APIs need it. */
+            if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
+                qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                                _("failed to update or add vcpupin xml of "
+                                  "a running domain"));
+                goto cleanup;
+            }
+
+            /* Configure the corresponding cpuset cgroup before set affinity. */
+            if (qemuCgroupControllerActive(driver,
+                                           VIR_CGROUP_CONTROLLER_CPUSET)) {
+                if (virCgroupForDomain(driver->cgroup, vm->def->name,
+                                       &cgroup_dom, 0) == 0) {
+                    if (virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0) == 0) {
+                        if (qemuSetupCgroupVcpuPin(cgroup_vcpu, vm->def, vcpu) < 0) {
+                            qemuReportError(VIR_ERR_OPERATION_INVALID, "%s %d",
+                                            _("failed to set cpuset.cpus in cgroup"
+                                              " for vcpu"), vcpu);
+                            goto cleanup;
+                        }
+                    }
+                }
+            }
+
             if (virProcessInfoSetAffinity(priv->vcpupids[vcpu],
-                                          cpumap, maplen, maxcpu) < 0)
+                                          cpumap, maplen, maxcpu) < 0) {
+                qemuReportError(VIR_ERR_SYSTEM_ERROR, "%s %d",
+                                _("failed to set cpu affinity for vcpu"),
+                                vcpu);
                 goto cleanup;
+            }
         } else {
             qemuReportError(VIR_ERR_OPERATION_INVALID,
                             "%s", _("cpu affinity is not supported"));
@@ -3621,13 +3651,6 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
                                   "a running domain"));
                 goto cleanup;
             }
-        } else {
-            if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
-                qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                                _("failed to update or add vcpupin xml of "
-                                  "a running domain"));
-                goto cleanup;
-            }
         }
 
         if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0)
@@ -3659,6 +3682,10 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
     ret = 0;
 
 cleanup:
+    if (cgroup_vcpu)
+        virCgroupFree(&cgroup_vcpu);
+    if (cgroup_dom)
+        virCgroupFree(&cgroup_dom);
     if (vm)
         virDomainObjUnlock(vm);
     return ret;
diff --git a/src/util/cgroup.c b/src/util/cgroup.c
index c5dddc1..ba3153c 100644
--- a/src/util/cgroup.c
+++ b/src/util/cgroup.c
@@ -532,7 +532,8 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group,
         /* We need to control cpu bandwidth for each vcpu now */
         if ((flags & VIR_CGROUP_VCPU) &&
             (i != VIR_CGROUP_CONTROLLER_CPU &&
-             i != VIR_CGROUP_CONTROLLER_CPUACCT)) {
+             i != VIR_CGROUP_CONTROLLER_CPUACCT &&
+             i != VIR_CGROUP_CONTROLLER_CPUSET)) {
             /* treat it as unmounted and we can use virCgroupAddTask */
             VIR_FREE(group->controllers[i].mountPoint);
             continue;
@@ -1337,6 +1338,38 @@ int virCgroupGetCpusetMems(virCgroupPtr group, char **mems)
 }
 
 /**
+ * virCgroupSetCpusetCpus:
+ *
+ * @group: The cgroup to set cpuset.cpus for
+ * @cpus: the cpus to set
+ *
+ * Retuens: 0 on success
+ */
+int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus)
+{
+    return virCgroupSetValueStr(group,
+                                VIR_CGROUP_CONTROLLER_CPUSET,
+                                "cpuset.cpus",
+                                cpus);
+}
+
+/**
+ * virCgroupGetCpusetCpus:
+ *
+ * @group: The cgroup to get cpuset.cpus for
+ * @cpus: the cpus to get
+ *
+ * Retuens: 0 on success
+ */
+int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus)
+{
+    return virCgroupGetValueStr(group,
+                                VIR_CGROUP_CONTROLLER_CPUSET,
+                                "cpuset.cpus",
+                                cpus);
+}
+
+/**
  * virCgroupDenyAllDevices:
  *
  * @group: The cgroup to deny all permissions, for all devices
diff --git a/src/util/cgroup.h b/src/util/cgroup.h
index 308ea47..1e01cbd 100644
--- a/src/util/cgroup.h
+++ b/src/util/cgroup.h
@@ -133,6 +133,9 @@ int virCgroupGetFreezerState(virCgroupPtr group, char **state);
 int virCgroupSetCpusetMems(virCgroupPtr group, const char *mems);
 int virCgroupGetCpusetMems(virCgroupPtr group, char **mems);
 
+int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus);
+int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus);
+
 int virCgroupRemove(virCgroupPtr group);
 
 void virCgroupFree(virCgroupPtr *group);
-- 
1.7.3.1

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list


[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]