Add cgroup for migation thread and implement
qemuDomainPinMigrationThread to pin migraiton thread to
given cpumap.
Signed-off-by: zhengchuan<zhengchuan@xxxxxxxxxx>
Signed-off-by: Jiang Jiacheng <jiangjiacheng@xxxxxxxxxx>
---
src/conf/domain_conf.c | 9 ++++++
src/conf/domain_conf.h | 10 +++++++
src/conf/virconftypes.h | 2 ++
src/libvirt_private.syms | 1 +
src/qemu/qemu_domain.c | 1 +
src/qemu/qemu_domain.h | 1 +
src/qemu/qemu_driver.c | 56 ++++++++++++++++++++++++++++++++++++
src/qemu/qemu_process.c | 61 ++++++++++++++++++++++++++++++++++++++++
src/qemu/qemu_process.h | 10 +++++++
src/util/vircgroup.c | 3 ++
src/util/vircgroup.h | 1 +
11 files changed, 155 insertions(+)
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index f7f9ec3c0a..68200dbbab 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -30883,3 +30883,12 @@ virDomainWatchdogDefFind(const virDomainDef *def,
return -1;
}
+
+void
+virDomainMigrationIDDefFree(virDomainMigrationIDDef *def)
+{
+ if (!def)
+ return;
+ virBitmapFree(def->cpumask);
+ VIR_FREE(def);
+}
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 21ea4a48b3..fff846f26f 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -4469,3 +4469,13 @@ virDomainObjGetMessages(virDomainObj *vm,
bool
virDomainDefHasSpiceGraphics(const virDomainDef *def);
+
+struct _virDomainMigrationIDDef {
+ bool autofill;
+ int thread_id;
+ virBitmap *cpumask;
+ virDomainThreadSchedParam sched;
+};
+
+void
+virDomainMigrationIDDefFree(virDomainMigrationIDDef *def);
diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h
index e07f967814..84a28a08b8 100644
--- a/src/conf/virconftypes.h
+++ b/src/conf/virconftypes.h
@@ -261,3 +261,5 @@ typedef struct _virDomainXMLOption virDomainXMLOption;
typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks;
typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts;
+
+typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef;
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index d5b1b9cb72..b5d9aa62ae 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -525,6 +525,7 @@ virDomainMemoryModelTypeToString;
virDomainMemoryRemove;
virDomainMemorySourceTypeFromString;
virDomainMemorySourceTypeToString;
+virDomainMigrationIDDefFree;
virDomainMouseModeTypeFromString;
virDomainMouseModeTypeToString;
virDomainNetAllocateActualDevice;
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index ba8d1055ae..3d497dd5d0 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -1853,6 +1853,7 @@ qemuDomainObjPrivateDataClear(qemuDomainObjPrivate *priv)
/* remove automatic pinning data */
g_clear_pointer(&priv->autoNodeset, virBitmapFree);
g_clear_pointer(&priv->autoCpuset, virBitmapFree);
+ g_clear_pointer(&priv->pcpumap, virBitmapFree);
g_clear_pointer(&priv->pciaddrs, virDomainPCIAddressSetFree);
g_clear_pointer(&priv->usbaddrs, virDomainUSBAddressSetFree);
g_clear_pointer(&priv->origCPU, virCPUDefFree);
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 42115a20ef..af986b5423 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -171,6 +171,7 @@ struct _qemuDomainObjPrivate {
/* Bitmaps below hold data from the auto NUMA feature */
virBitmap *autoNodeset;
virBitmap *autoCpuset;
+ virBitmap *pcpumap;
bool signalIOError; /* true if the domain condition should be signalled on
I/O error */
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index c546e35953..2b9a952c85 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -20581,6 +20581,61 @@ qemuDomainFDAssociate(virDomainPtr domain,
return ret;
}
+static int
+qemuDomainPinMigrationThread(virDomainPtr dom,
+ unsigned char *cpumap,
+ int maplen)
+{
+ int ret = -1;
+ virQEMUDriver *driver = dom->conn->privateData;
+ g_autoptr(virQEMUDriverConfig) cfg = NULL;
+ virDomainObj *vm = NULL;
+ g_autoptr(virBitmap) pcpumap = NULL;
+ qemuDomainObjPrivate *priv = NULL;
+ qemuDomainMigThreadInfo **migthreads = NULL;
+ size_t i;
+
+ cfg = virQEMUDriverGetConfig(driver);
+
+ if (!(vm = qemuDomainObjFromDomain(dom)))
+ goto cleanup;
+
+ priv = vm->privateData;
+
+ if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0)
+ goto cleanup;
+
+ if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0)
+ goto cleanup;
+
+ if (!(pcpumap = virBitmapNewData(cpumap, maplen)))
+ goto endjob;
+
+ if (virBitmapIsAllClear(pcpumap)) {
+ virReportError(VIR_ERR_INVALID_ARG, "%s",
+ _("Empty migration thread cpumap list for pinning"));
+ goto endjob;
+ }
+
+ virBitmapFree(priv->pcpumap);
+ priv->pcpumap = virBitmapNewCopy(pcpumap);
+ migthreads = priv->migThreadsInfo;
+
+ if (migthreads && pcpumap) {
+ for (i = 0; i < priv->migThreadCount; i++) {
+ qemuProcessSetMigThreadAffinity(priv, vm, migthreads[i]->thread_id, pcpumap);
+ }
+ }
+
+ ret = 0;
+
+ endjob:
+ virDomainObjEndJob(vm);
+
+ cleanup:
+ virDomainObjEndAPI(&vm);
+ return ret;
+}
static virHypervisorDriver qemuHypervisorDriver = {
.name = QEMU_DRIVER_NAME,
@@ -20831,6 +20886,7 @@ static virHypervisorDriver qemuHypervisorDriver = {
.domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */
.domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */
.domainFDAssociate = qemuDomainFDAssociate, /* 9.0.0 */
+ .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */
};
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 9a612ca443..946aeb81b1 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -9547,3 +9547,64 @@ qemuProcessQMPStart(qemuProcessQMP *proc)
return 0;
}
+
+int
+qemuProcessSetupMigration(virDomainObj *vm,
+ virDomainMigrationIDDef *migration)
+{
+ return qemuProcessSetupPid(vm, migration->thread_id,
+ VIR_CGROUP_THREAD_MIGRATION_THREAD,
+ 0,
+ vm->def->cputune.emulatorpin,
+ vm->def->cputune.emulator_period,
+ vm->def->cputune.emulator_quota,
+ &migration->sched);
+}
+
+/*
+ * In order to set migration thread affinity when vm is migrating,
+ * we should create the cgroup for migration thread.
+ */
+void
+qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv,
+ virDomainObj *vm,
+ int mpid,
+ virBitmap *pcpumap)
+{
+ virDomainMigrationIDDef *migration = NULL;
+ int migration_id = 0;
+ virCgroup *cgroup_migthread = NULL;
+
+ if (!pcpumap)
+ return;
+
+ migration = g_new0(virDomainMigrationIDDef, 1);
+ migration->thread_id = mpid;
+ if (qemuProcessSetupMigration(vm, migration) < 0) {
+ VIR_ERROR(_("fail to setup migration cgroup"));
+ goto cleanup;
+ }
+
+ if (virCgroupHasController(priv->cgroup,
+ VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD,
+ migration_id, false, &cgroup_migthread) < 0)
+ goto cleanup;
+
+ if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) {
+ virReportError(VIR_ERR_OPERATION_INVALID,
+ _("failed to set cpuset.cpus in cgroup"
+ " for migration%d thread"), migration_id);
+ goto cleanup;
+ }
+ }
+
+ if (virProcessSetAffinity(mpid, pcpumap, false) < 0)
+ VIR_WARN("failed to set affinity in migration");
+
+ cleanup:
+ if (cgroup_migthread)
+ virCgroupFree(cgroup_migthread);
+ virDomainMigrationIDDefFree(migration);
+ return;
+}
diff --git a/src/qemu/qemu_process.h b/src/qemu/qemu_process.h
index b171f0464c..2af1e2ee87 100644
--- a/src/qemu/qemu_process.h
+++ b/src/qemu/qemu_process.h
@@ -235,3 +235,13 @@ bool qemuProcessRebootAllowed(const virDomainDef *def);
void qemuProcessCleanupMigrationJob(virQEMUDriver *driver,
virDomainObj *vm);
+
+int
+qemuProcessSetupMigration(virDomainObj *vm,
+ virDomainMigrationIDDef *migration);
+
+void
+qemuProcessSetMigThreadAffinity(qemuDomainObjPrivate *priv,
+ virDomainObj *vm,
+ int mpid,
+ virBitmap *pcpumap);
diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
index 73675b4478..d4307c0d72 100644
--- a/src/util/vircgroup.c
+++ b/src/util/vircgroup.c
@@ -1123,6 +1123,9 @@ virCgroupNewThread(virCgroup *domain,
case VIR_CGROUP_THREAD_IOTHREAD:
name = g_strdup_printf("iothread%d", id);
break;
+ case VIR_CGROUP_THREAD_MIGRATION_THREAD:
+ name = g_strdup_printf("migthread%d", id);
+ break;
case VIR_CGROUP_THREAD_LAST:
virReportError(VIR_ERR_INTERNAL_ERROR,
_("unexpected name value %d"), nameval);
diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h
index adf3850b22..c5f6ddd7a6 100644
--- a/src/util/vircgroup.h
+++ b/src/util/vircgroup.h
@@ -53,6 +53,7 @@ typedef enum {
VIR_CGROUP_THREAD_VCPU = 0,
VIR_CGROUP_THREAD_EMULATOR,
VIR_CGROUP_THREAD_IOTHREAD,
+ VIR_CGROUP_THREAD_MIGRATION_THREAD,
VIR_CGROUP_THREAD_LAST
} virCgroupThreadName;
--
2.33.0
