Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1786303 Signed-off-by: Michal Privoznik <mprivozn@xxxxxxxxxx> --- src/conf/numa_conf.c | 7 + src/qemu/qemu_command.c | 183 ++++++++++++++++++ .../numatune-hmat.x86_64-latest.args | 53 +++++ tests/qemuxml2argvtest.c | 1 + 4 files changed, 244 insertions(+) create mode 100644 tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c index b35e5040fc..7fbbe0f52e 100644 --- a/src/conf/numa_conf.c +++ b/src/conf/numa_conf.c @@ -1904,6 +1904,13 @@ virDomainNumaGetNodeInitiator(const virDomainNuma *numa, if (!numa || node >= numa->nmem_nodes) return -1; + /* A NUMA node which has at least one vCPU is initiator to itself by + * definition. */ + if (numa->mem_nodes[node].cpumask) + return node; + + /* For the rest, "NUMA node that has best performance (the lowest + * latency or largest bandwidth) to this NUMA node." */ for (i = 0; i < numa->nlatencies; i++) { const virDomainNumaLatency *l = &numa->latencies[i]; diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index 628ff970bd..b28c43f110 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -6932,6 +6932,16 @@ qemuBuildMachineCommandLine(virCommandPtr cmd, virBufferAsprintf(&buf, ",pflash1=%s", priv->pflash1->nodeformat); } + if (virDomainNumaHasHMAT(def->numa)) { + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("HMAT is not supported with this QEMU")); + return -1; + } + + virBufferAddLit(&buf, ",hmat=on"); + } + virCommandAddArgBuffer(cmd, &buf); return 0; @@ -7114,6 +7124,134 @@ qemuBuildIOThreadCommandLine(virCommandPtr cmd, } +static int +qemuBuilNumaCellCache(virCommandPtr cmd, + const virDomainDef *def, + size_t cell) +{ + size_t ncaches = virDomainNumaGetNodeCacheCount(def->numa, cell); + size_t i; + + if (ncaches == 0) + return 0; + + for (i = 0; i < ncaches; i++) { + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + unsigned int level; + unsigned int size; + unsigned int line; + virDomainCacheAssociativity associativity; + virDomainCachePolicy policy; + + if (virDomainNumaGetNodeCache(def->numa, cell, i, + &level, &size, &line, + &associativity, &policy) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to format NUMA node cache")); + return -1; + } + + virBufferAsprintf(&buf, + "hmat-cache,node-id=%zu,size=%uK,level=%u", + cell, size, level); + + switch (associativity) { + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_NONE: + virBufferAddLit(&buf, ",associativity=none"); + break; + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_DIRECT: + virBufferAddLit(&buf, ",associativity=direct"); + break; + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_FULL: + virBufferAddLit(&buf, ",associativity=complex"); + break; + case VIR_DOMAIN_CACHE_ASSOCIATIVITY_LAST: + break; + } + + switch (policy) { + case VIR_DOMAIN_CACHE_POLICY_NONE: + virBufferAddLit(&buf, ",policy=none"); + break; + case VIR_DOMAIN_CACHE_POLICY_WRITEBACK: + virBufferAddLit(&buf, ",policy=write-back"); + break; + case VIR_DOMAIN_CACHE_POLICY_WRITETHROUGH: + virBufferAddLit(&buf, ",policy=write-through"); + break; + case VIR_DOMAIN_CACHE_POLICY_LAST: + break; + } + + if (line > 0) + virBufferAsprintf(&buf, ",line=%u", line); + + virCommandAddArg(cmd, "-numa"); + virCommandAddArgBuffer(cmd, &buf); + } + + return 0; +} + + +VIR_ENUM_DECL(qemuDomainMemoryHierarchy); +VIR_ENUM_IMPL(qemuDomainMemoryHierarchy, + 4, /* Maximum level of cache */ + "memory", /* Special case, whole memory not specific cache */ + "first-level", + "second-level", + "third-level"); + +static int +qemuBuildNumaHMATCommandLine(virCommandPtr cmd, + const virDomainDef *def) +{ + size_t nlatencies; + size_t i; + + if (!def->numa) + return 0; + + nlatencies = virDomainNumaGetLatenciesCount(def->numa); + for (i = 0; i < nlatencies; i++) { + g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; + virDomainNumaLatencyType type; + unsigned int initiator; + unsigned int target; + unsigned int cache; + virDomainMemoryLatency accessType; + unsigned long value; + const char *hierarchyStr; + const char *accessStr; + + if (virDomainNumaGetLatency(def->numa, i, + &type, &initiator, &target, + &cache, &accessType, &value) < 0) + return -1; + + hierarchyStr = qemuDomainMemoryHierarchyTypeToString(cache); + accessStr = virDomainMemoryLatencyTypeToString(accessType); + virBufferAsprintf(&buf, + "hmat-lb,initiator=%u,target=%u,hierarchy=%s,data-type=%s-", + initiator, target, hierarchyStr, accessStr); + + switch (type) { + case VIR_DOMAIN_NUMA_LATENCY_TYPE_LATENCY: + virBufferAsprintf(&buf, "latency,latency=%lu", value); + break; + case VIR_DOMAIN_NUMA_LATENCY_TYPE_BANDWIDTH: + virBufferAsprintf(&buf, "bandwidth,bandwidth=%luK", value); + break; + } + + virCommandAddArg(cmd, "-numa"); + virCommandAddArgBuffer(cmd, &buf); + } + + return 0; +} + + static int qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, virDomainDefPtr def, @@ -7126,9 +7264,11 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, char *next = NULL; virBufferPtr nodeBackends = NULL; bool needBackend = false; + bool hmat = false; int rc; int ret = -1; size_t ncells = virDomainNumaGetNodeCount(def->numa); + ssize_t masterInitiator = -1; if (!virDomainNumatuneNodesetIsAvailable(def->numa, priv->autoNodeset)) goto cleanup; @@ -7138,6 +7278,16 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, def->os.machine)) needBackend = true; + if (virDomainNumaHasHMAT(def->numa)) { + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA_HMAT)) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("HMAT is not supported with this QEMU")); + goto cleanup; + } + needBackend = true; + hmat = true; + } + if (VIR_ALLOC_N(nodeBackends, ncells) < 0) goto cleanup; @@ -7166,8 +7316,22 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, qemuBuildMemPathStr(def, cmd, priv) < 0) goto cleanup; + for (i = 0; i < ncells; i++) { + if (virDomainNumaGetNodeCpumask(def->numa, i)) { + masterInitiator = i; + break; + } + } + + if (masterInitiator) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("At least one NUMA node has to have CPUs")); + goto cleanup; + } + for (i = 0; i < ncells; i++) { virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i); + ssize_t initiator = virDomainNumaGetNodeInitiator(def->numa, i); if (needBackend) { virCommandAddArg(cmd, "-object"); @@ -7192,6 +7356,13 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, } } + if (hmat) { + if (initiator < 0) + initiator = masterInitiator; + + virBufferAsprintf(&buf, ",initiator=%zd", initiator); + } + if (needBackend) virBufferAsprintf(&buf, ",memdev=ram-node%zu", i); else @@ -7217,6 +7388,18 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg, } } + if (hmat) { + if (qemuBuildNumaHMATCommandLine(cmd, def) < 0) + goto cleanup; + + /* This can't be moved into any of the loops above, + * because hmat-cache can be specified only after hmat-lb. */ + for (i = 0; i < ncells; i++) { + if (qemuBuilNumaCellCache(cmd, def, i) < 0) + goto cleanup; + } + } + ret = 0; cleanup: diff --git a/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args new file mode 100644 index 0000000000..c52015caa8 --- /dev/null +++ b/tests/qemuxml2argvdata/numatune-hmat.x86_64-latest.args @@ -0,0 +1,53 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/tmp/lib/domain--1-QEMUGuest \ +USER=test \ +LOGNAME=test \ +XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \ +XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \ +XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \ +QEMU_AUDIO_DRV=none \ +/usr/bin/qemu-system-x86_64 \ +-name guest=QEMUGuest,debug-threads=on \ +-S \ +-object secret,id=masterKey0,format=raw,\ +file=/tmp/lib/domain--1-QEMUGuest/master-key.aes \ +-machine pc,accel=tcg,usb=off,dump-guest-core=off,hmat=on \ +-cpu qemu64 \ +-m 12288 \ +-overcommit mem-lock=off \ +-smp 12,sockets=12,cores=1,threads=1 \ +-object memory-backend-ram,id=ram-node0,size=2147483648 \ +-numa node,nodeid=0,cpus=0-3,initiator=0,memdev=ram-node0 \ +-object memory-backend-ram,id=ram-node1,size=2147483648 \ +-numa node,nodeid=1,cpus=4-7,initiator=1,memdev=ram-node1 \ +-object memory-backend-ram,id=ram-node2,size=2147483648 \ +-numa node,nodeid=2,cpus=8-11,initiator=2,memdev=ram-node2 \ +-object memory-backend-ram,id=ram-node3,size=2147483648 \ +-numa node,nodeid=3,initiator=0,memdev=ram-node3 \ +-object memory-backend-ram,id=ram-node4,size=2147483648 \ +-numa node,nodeid=4,initiator=0,memdev=ram-node4 \ +-object memory-backend-ram,id=ram-node5,size=2147483648 \ +-numa node,nodeid=5,initiator=0,memdev=ram-node5 \ +-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-latency,\ +latency=5 \ +-numa hmat-lb,initiator=0,target=0,hierarchy=first-level,\ +data-type=access-latency,latency=10 \ +-numa hmat-lb,initiator=0,target=0,hierarchy=memory,data-type=access-bandwidth,\ +bandwidth=204800K \ +-numa hmat-cache,node-id=0,size=10K,level=1,associativity=direct,\ +policy=write-back,line=8 \ +-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \ +-display none \ +-no-user-config \ +-nodefaults \ +-chardev socket,id=charmonitor,fd=1729,server,nowait \ +-mon chardev=charmonitor,id=monitor,mode=control \ +-rtc base=utc \ +-no-shutdown \ +-boot strict=on \ +-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 \ +-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3 \ +-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,\ +resourcecontrol=deny \ +-msg timestamp=on diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 98d9a7b5b2..3fdcd41317 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1913,6 +1913,7 @@ mymain(void) DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST); DO_TEST("numatune-no-vcpu", NONE); + DO_TEST_CAPS_LATEST("numatune-hmat"); DO_TEST("numatune-auto-nodeset-invalid", NONE); DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM, -- 2.26.2
