This patch enables cpuset cgroup, and synchronous vcpupin info set by sched_setaffinity() to cgroup.
Signed-off-by: Tang Chen <tangchen@xxxxxxxxxxxxxx>
---
src/qemu/qemu_cgroup.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_cgroup.h | 2 ++
src/qemu/qemu_driver.c | 43 +++++++++++++++++++++++++++++++++++--------
src/util/cgroup.c | 35 ++++++++++++++++++++++++++++++++++-
src/util/cgroup.h | 3 +++
5 files changed, 121 insertions(+), 9 deletions(-)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 7c6ef33..a123a00 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -471,11 +471,50 @@ cleanup:
return -1;
}
+int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
+ int vcpuid)
+{
+ int i, rc;
+ char *new_cpus = NULL;
+
+ if (vcpuid < 0 || vcpuid >= def->vcpus) {
+ virReportSystemError(EINVAL,
+ _("%s: %d"), _("invalid vcpuid"), vcpuid);
+ return -1;
+ }
+
+ for (i = 0; i < def->cputune.nvcpupin; i++) {
+ if (vcpuid == def->cputune.vcpupin[i]->vcpuid) {
+ new_cpus = virDomainCpuSetFormat(def->cputune.vcpupin[i]->cpumask,
+ VIR_DOMAIN_CPUMASK_LEN);
+ if (!new_cpus) {
+ qemuReportError(VIR_ERR_INTERNAL_ERROR,
+ _("failed to convert cpu mask"));
+ goto cleanup;
+ }
+ rc = virCgroupSetCpusetCpus(cgroup, new_cpus);
+ if (rc < 0) {
+ virReportSystemError(-rc,
+ _("%s"), _("Unable to set cpuset.cpus"));
+ goto cleanup;
+ }
+ }
+ }
+ VIR_FREE(new_cpus);
+ return 0;
+
+cleanup:
+ if (new_cpus)
+ VIR_FREE(new_cpus);
+ return -1;
+}
+
int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
{
virCgroupPtr cgroup = NULL;
virCgroupPtr cgroup_vcpu = NULL;
qemuDomainObjPrivatePtr priv = vm->privateData;
+ virDomainDefPtr def = vm->def;
int rc;
unsigned int i;
unsigned long long period = vm->def->cputune.period;
@@ -553,6 +592,14 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
}
}
+ /* Set vcpupin in cgroup if vcpupin xml is provided */
+ if (def->cputune.nvcpupin) {
+ if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (qemuSetupCgroupVcpuPin(cgroup_vcpu, def, i) < 0)
+ goto cleanup;
+ }
+ }
+
virCgroupFree(&cgroup_vcpu);
}
diff --git a/src/qemu/qemu_cgroup.h b/src/qemu/qemu_cgroup.h
index 92eff68..dbf783a 100644
--- a/src/qemu/qemu_cgroup.h
+++ b/src/qemu/qemu_cgroup.h
@@ -52,6 +52,8 @@ int qemuSetupCgroup(struct qemud_driver *driver,
int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup,
unsigned long long period,
long long quota);
+int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup, virDomainDefPtr def,
+ int vcpuid);
int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm);
int qemuSetupCgroupForHypervisor(struct qemud_driver *driver,
virDomainObjPtr vm);
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 165d1c0..9333d1c 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -3511,6 +3511,8 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
struct qemud_driver *driver = dom->conn->privateData;
virDomainObjPtr vm;
virDomainDefPtr persistentDef = NULL;
+ virCgroupPtr cgroup_dom = NULL;
+ virCgroupPtr cgroup_vcpu = NULL;
int maxcpu, hostcpus;
virNodeInfo nodeinfo;
int ret = -1;
@@ -3565,9 +3567,37 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
if (flags & VIR_DOMAIN_AFFECT_LIVE) {
if (priv->vcpupids != NULL) {
+ /* Add config to vm->def first, because cgroup APIs need it. */
+ if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
+ qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to update or add vcpupin xml of "
+ "a running domain"));
+ goto cleanup;
+ }
+
+ /* Configure the corresponding cpuset cgroup before set affinity. */
+ if (qemuCgroupControllerActive(driver,
+ VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (virCgroupForDomain(driver->cgroup, vm->def->name,
+ &cgroup_dom, 0) == 0) {
+ if (virCgroupForVcpu(cgroup_dom, vcpu, &cgroup_vcpu, 0) == 0) {
+ if (qemuSetupCgroupVcpuPin(cgroup_vcpu, vm->def, vcpu) < 0) {
+ qemuReportError(VIR_ERR_OPERATION_INVALID, "%s %d",
+ _("failed to set cpuset.cpus in cgroup"
+ " for vcpu"), vcpu);
+ goto cleanup;
+ }
+ }
+ }
+ }
+
if (virProcessInfoSetAffinity(priv->vcpupids[vcpu],
- cpumap, maplen, maxcpu) < 0)
+ cpumap, maplen, maxcpu) < 0) {
+ qemuReportError(VIR_ERR_SYSTEM_ERROR, "%s %d",
+ _("failed to set cpu affinity for vcpu"),
+ vcpu);
goto cleanup;
+ }
} else {
qemuReportError(VIR_ERR_OPERATION_INVALID,
"%s", _("cpu affinity is not supported"));
@@ -3581,13 +3611,6 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
"a running domain"));
goto cleanup;
}
- } else {
- if (virDomainVcpuPinAdd(vm->def, cpumap, maplen, vcpu) < 0) {
- qemuReportError(VIR_ERR_INTERNAL_ERROR, "%s",
- _("failed to update or add vcpupin xml of "
- "a running domain"));
- goto cleanup;
- }
}
if (virDomainSaveStatus(driver->caps, driver->stateDir, vm) < 0)
@@ -3619,6 +3642,10 @@ qemudDomainPinVcpuFlags(virDomainPtr dom,
ret = 0;
cleanup:
+ if (cgroup_vcpu)
+ virCgroupFree(&cgroup_vcpu);
+ if (cgroup_dom)
+ virCgroupFree(&cgroup_dom);
if (vm)
virDomainObjUnlock(vm);
return ret;
diff --git a/src/util/cgroup.c b/src/util/cgroup.c
index 724cc6e..fd2e84d 100644
--- a/src/util/cgroup.c
+++ b/src/util/cgroup.c
@@ -530,7 +530,8 @@ static int virCgroupMakeGroup(virCgroupPtr parent, virCgroupPtr group,
continue;
/* We need to control cpu bandwidth for each vcpu now */
- if ((flags & VIR_CGROUP_VCPU) && (i != VIR_CGROUP_CONTROLLER_CPU)) {
+ if ((flags & VIR_CGROUP_VCPU) && (i != VIR_CGROUP_CONTROLLER_CPU) &&
+ (i != VIR_CGROUP_CONTROLLER_CPUSET)) {
/* treat it as unmounted and we can use virCgroupAddTask */
VIR_FREE(group->controllers[i].mountPoint);
continue;
@@ -1335,6 +1336,38 @@ int virCgroupGetCpusetMems(virCgroupPtr group, char **mems)
}
/**
+ * virCgroupSetCpusetCpus:
+ *
+ * @group: The cgroup to set cpuset.cpus for
+ * @cpus: the cpus to set
+ *
+ * Retuens: 0 on success
+ */
+int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus)
+{
+ return virCgroupSetValueStr(group,
+ VIR_CGROUP_CONTROLLER_CPUSET,
+ "cpuset.cpus",
+ cpus);
+}
+
+/**
+ * virCgroupGetCpusetCpus:
+ *
+ * @group: The cgroup to get cpuset.cpus for
+ * @cpus: the cpus to get
+ *
+ * Retuens: 0 on success
+ */
+int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus)
+{
+ return virCgroupGetValueStr(group,
+ VIR_CGROUP_CONTROLLER_CPUSET,
+ "cpuset.cpus",
+ cpus);
+}
+
+/**
* virCgroupDenyAllDevices:
*
* @group: The cgroup to deny all permissions, for all devices
diff --git a/src/util/cgroup.h b/src/util/cgroup.h
index 2419ef6..beade4e 100644
--- a/src/util/cgroup.h
+++ b/src/util/cgroup.h
@@ -131,6 +131,9 @@ int virCgroupGetFreezerState(virCgroupPtr group, char **state);
int virCgroupSetCpusetMems(virCgroupPtr group, const char *mems);
int virCgroupGetCpusetMems(virCgroupPtr group, char **mems);
+int virCgroupSetCpusetCpus(virCgroupPtr group, const char *cpus);
+int virCgroupGetCpusetCpus(virCgroupPtr group, char **cpus);
+
int virCgroupRemove(virCgroupPtr group);
void virCgroupFree(virCgroupPtr *group);
--
1.7.3.1
--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list