This patch enables cpuset cgroup, and synchronous vcpupin info set by sched_setaffinity() to cgroup.
Signed-off-by: Tang Chen <tangchen@xxxxxxxxxxxxxx>
---
src/libvirt_private.syms | 2 +
src/qemu/qemu_cgroup.c | 51 ++++++++++++++++++++++++++++++++++++++++++++-
src/qemu/qemu_cgroup.h | 2 +
src/qemu/qemu_driver.c | 43 +++++++++++++++++++++++++++++++-------
src/util/cgroup.c | 35 ++++++++++++++++++++++++++++++-
src/util/cgroup.h | 3 ++
6 files changed, 125 insertions(+), 11 deletions(-)
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 6ff1a3b..88cc37a 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -77,6 +77,7 @@ virCgroupGetCpuShares;
virCgroupGetCpuacctPercpuUsage;
virCgroupGetCpuacctStat;
virCgroupGetCpuacctUsage;
+virCgroupGetCpusetCpus;
virCgroupGetCpusetMems;
virCgroupGetFreezerState;
virCgroupGetMemSwapHardLimit;
@@ -95,6 +96,7 @@ virCgroupSetBlkioWeight;
virCgroupSetCpuCfsPeriod;
virCgroupSetCpuCfsQuota;
virCgroupSetCpuShares;
+virCgroupSetCpusetCpus;
virCgroupSetCpusetMems;
virCgroupSetFreezerState;
virCgroupSetMemSwapHardLimit;
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index e69ef5b..1085478 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -473,18 +473,57 @@ cleanup:
rc = virCgroupSetCpuCfsPeriod(cgroup, old_period);
if (rc < 0)
virReportSystemError(-rc,
- _("%s"),
- "Unable to rollback cpu bandwidth period");
+ "%s",
+ _("Unable to rollback cpu bandwidth period"));
}
return -1;
}
+int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
+ int vcpuid)
+{
+ int i, rc;
+ char *new_cpus = NULL;
+
+ if (vcpuid < 0 || vcpuid >= def->vcpus) {
+ virReportSystemError(EINVAL,
+ "%s: %d", _("invalid vcpuid"), vcpuid);
+ return -1;
+ }
+
+ for (i = 0; i < def->cputune.nvcpupin; i++) {
+ if (vcpuid == def->cputune.vcpupin[i]->vcpuid) {
+ new_cpus = virDomainCpuSetFormat(def->cputune.vcpupin[i]->cpumask,
+ VIR_DOMAIN_CPUMASK_LEN);
+ if (!new_cpus) {
+ qemuReportError(VIR_ERR_INTERNAL_ERROR,
+ _("failed to convert cpu mask"));
+ goto cleanup;
+ }
+ rc = virCgroupSetCpusetCpus(cgroup, new_cpus);
+ if (rc < 0) {
+ virReportSystemError(-rc,
+ "%s", _("Unable to set cpuset.cpus"));
+ goto cleanup;
+ }
+ }
+ }
+ VIR_FREE(new_cpus);
+ return 0;
+
+cleanup:
+ if (new_cpus)
+ VIR_FREE(new_cpus);
+ return -1;
+}
+
int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
{
virCgroupPtr cgroup = NULL;
virCgroupPtr cgroup_vcpu = NULL;
qemuDomainObjPrivatePtr priv = vm->privateData;
+ virDomainDefPtr def = vm->def;
int rc;
unsigned int i;
unsigned long long period = vm->def->cputune.period;
@@ -556,6 +595,14 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
}
}
+ /* Set vcpupin in cgroup if vcpupin xml is provided */
+ if (def->cputune.nvcpupin) {
+ if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (qemuSetupCgroupVcpuPin(cgroup_vcpu, def, i) < 0)
+ goto cleanup;
+ }
+ }
+
virCgroupFree(&cgroup_vcpu);
}
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index cf0d383..91d5632 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -53,6 +53,8 @@ int qemuSetupCgroup(struct qemud_driver *driver,
int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
unsigned long long period,
long long quota);
+int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
+ int vcpuid);
int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm);
int qemuSetupCgroupForHypervisor(struct qemud_driver *driver,
virDomainObjPtr vm);
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index d3f74d2..b0eef80 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -3551,6 +3551,8 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
struct qemud_driver *driver = dom->conn->privateData;
virDomainObjPtr vm;
virDomainDefPtr persistentDef = NULL;
+ virCgroupPtr cgroup_dom = NULL;
+ virCgroupPtr cgroup_vcpu = NULL;
int maxcpu, hostcpus;
virNodeInfo nodeinfo;
int ret = -1;
@@ -3605,9 +3607,37 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
if (flags & VIR_DOMAIN_AFFECT_LIVE) {
if (priv->vcpupids != NULL) {
+ /* Add config to vm->def first, because cgroup APIs need it. */
+ if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
+ qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to update or add vcpupin xml of "
+ "a running domain"));
+ goto cleanup;
+ }
+
+ /* Configure the corresponding cpuset cgroup before set affinity. */
+ if (qemuCgroupControllerActive(driver,
+ VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (virCgroupForDomain(driver->cgroup, vm->def->name,
+ &cgroup_dom, 0) == 0) {
+ if (virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0) == 0) {
+ if (qemuSetupCgroupVcpuPin(cgroup_vcpu, vm->def, vcpu) < 0) {
+ qemuReportError(VIR_ERR_OPERATION_INVALID, "%s %d",
+ _("failed to set cpuset.cpus in cgroup"
+ " for vcpu"), vcpu);
+ goto cleanup;
+ }
+ }
+ }
+ }
+
if (virProcessInfoSetAffinity(priv->vcpupids[vcpu],
- cpumap, maplen, maxcpu) < 0)
+ cpumap, maplen, maxcpu) < 0) {
+ qemuReportError(VIR_ERR_SYSTEM_ERROR, "%s %d",
+ _("failed to set cpu affinity for vcpu"),
+ vcpu);
goto cleanup;
+ }
} else {
qemuReportError(VIR_ERR_OPERATION_INVALID,
"%s", _("cpu affinity is not supported"));
@@ -3621,13 +3651,6 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
"a running domain"));
goto cleanup;
}
- } else {
- if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
- qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
- _("failed to update or add vcpupin xml of "
- "a running domain"));
- goto cleanup;
- }
}
if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0)
@@ -3659,6 +3682,10 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
ret = 0;
cleanup:
+ if (cgroup_vcpu)
+ virCgroupFree(&cgroup_vcpu);
+ if (cgroup_dom)
+ virCgroupFree(&cgroup_dom);
if (vm)
virDomainObjUnlock(vm);
return ret;
diff --git a/src/util/cgroup.c b/src/util/cgroup.c
index c5dddc1..ba3153c 100644
--- a/src/util/cgroup.c
+++ b/src/util/cgroup.c
@@ -532,7 +532,8 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group,
/* We need to control cpu bandwidth for each vcpu now */
if ((flags & VIR_CGROUP_VCPU) &&
(i != VIR_CGROUP_CONTROLLER_CPU &&
- i != VIR_CGROUP_CONTROLLER_CPUACCT)) {
+ i != VIR_CGROUP_CONTROLLER_CPUACCT &&
+ i != VIR_CGROUP_CONTROLLER_CPUSET)) {
/* treat it as unmounted and we can use virCgroupAddTask */
VIR_FREE(group->controllers[i].mountPoint);
continue;
@@ -1337,6 +1338,38 @@ int virCgroupGetCpusetMems(virCgroupPtr group, char **mems)
}
/**
+ * virCgroupSetCpusetCpus:
+ *
+ * @group: The cgroup to set cpuset.cpus for
+ * @cpus: the cpus to set
+ *
+ * Retuens: 0 on success
+ */
+int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus)
+{
+ return virCgroupSetValueStr(group,
+ VIR_CGROUP_CONTROLLER_CPUSET,
+ "cpuset.cpus",
+ cpus);
+}
+
+/**
+ * virCgroupGetCpusetCpus:
+ *
+ * @group: The cgroup to get cpuset.cpus for
+ * @cpus: the cpus to get
+ *
+ * Retuens: 0 on success
+ */
+int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus)
+{
+ return virCgroupGetValueStr(group,
+ VIR_CGROUP_CONTROLLER_CPUSET,
+ "cpuset.cpus",
+ cpus);
+}
+
+/**
* virCgroupDenyAllDevices:
*
* @group: The cgroup to deny all permissions, for all devices
diff --git a/src/util/cgroup.h b/src/util/cgroup.h
index 308ea47..1e01cbd 100644
--- a/src/util/cgroup.h
+++ b/src/util/cgroup.h
@@ -133,6 +133,9 @@ int virCgroupGetFreezerState(virCgroupPtr group, char **state);
int virCgroupSetCpusetMems(virCgroupPtr group, const char *mems);
int virCgroupGetCpusetMems(virCgroupPtr group, char **mems);
+int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus);
+int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus);
+
int virCgroupRemove(virCgroupPtr group);
void virCgroupFree(virCgroupPtr *group);
--
1.7.3.1
--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list