Extracting capabilities from QEMU takes a notable amount of time
when all QEMU binaries are installed. Each system emulator
needs about 200-300ms multiplied by 26 binaries == ~5-8 seconds.
This change causes the QEMU driver to save an XML file containing
the content of the virQEMUCaps object instance in the cache
dir eg /var/cache/libvirt/qemu/capabilities/$SHA256(binarypath).xml
or $HOME/.cache/libvirt/qemu/cache/capabilities/$SHA256(binarypath).xml
We attempt to load this and only if it fails, do we fallback to
probing the QEMU binary. The timestamp of the file is compared to
the timestamp of the QEMU binary and discarded if the QEMU binary
is newer.
Signed-off-by: Daniel P. Berrange <berrange@xxxxxxxxxx>
---
src/qemu/qemu_capabilities.c | 412 ++++++++++++++++++++++++++++++++++++++++++-
src/qemu/qemu_capabilities.h | 2 +
src/qemu/qemu_driver.c | 1 +
3 files changed, 407 insertions(+), 8 deletions(-)
diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
index cae25e0..e3ed960 100644
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@@ -25,6 +25,7 @@
#include "qemu_capabilities.h"
#include "viralloc.h"
+#include "vircrypto.h"
#include "virlog.h"
#include "virerror.h"
#include "virfile.h"
@@ -44,6 +45,7 @@
#include <unistd.h>
#include <sys/wait.h>
#include <stdarg.h>
+#include <utime.h>
#define VIR_FROM_THIS VIR_FROM_QEMU
@@ -281,7 +283,7 @@ struct _virQEMUCapsCache {
virMutex lock;
virHashTablePtr binaries;
char *libDir;
- char *runDir;
+ char *cacheDir;
uid_t runUid;
gid_t runGid;
};
@@ -2339,6 +2341,386 @@ int virQEMUCapsProbeQMP(virQEMUCapsPtr qemuCaps,
}
+/*
+ * Parsing a doc that looks like
+ *
+ * <qemuCaps>
+ * <usedQMP/>
+ * <flag name='foo'/>
+ * <flag name='bar'/>
+ * ...
+ * <cpu name="pentium3"/>
+ * ...
+ * <machine name="pc-1.0" alias="pc" maxCpus="4"/>
+ * ...
+ * </qemuCaps>
+ */
+static int
+virQEMUCapsLoadCache(virQEMUCapsPtr qemuCaps, const char *filename)
+{
+ xmlDocPtr doc = NULL;
+ int ret = -1;
+ size_t i;
+ int n;
+ xmlNodePtr *nodes = NULL;
+ xmlXPathContextPtr ctxt = NULL;
+ char *str;
+
+ if (!(doc = virXMLParseFile(filename)))
+ goto cleanup;
+
+ if (!(ctxt = xmlXPathNewContext(doc))) {
+ virReportOOMError();
+ goto cleanup;
+ }
+
+ ctxt->node = xmlDocGetRootElement(doc);
+
+ if (STRNEQ((const char *)ctxt->node->name, "qemuCaps")) {
+ virReportError(VIR_ERR_XML_ERROR,
+ _("unexpected root element <%s>, "
+ "expecting <qemuCaps>"),
+ ctxt->node->name);
+ goto cleanup;
+ }
+
+ qemuCaps->usedQMP = virXPathBoolean("count(.//usedQMP) > 0",
+ ctxt) > 0;
+
+ if ((n = virXPathNodeSet(".//flag", ctxt, &nodes)) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to parse qemu capabilities flags"));
+ goto cleanup;
+ }
+ VIR_DEBUG("Got flags %d", n);
+ if (n > 0) {
+ for (i = 0; i < n; i++) {
+ int flag;
+ if (!(str = virXMLPropString(nodes[i], "name"))) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing flag name in QEMU capabilities cache"));
+ goto cleanup;
+ }
+ flag = virQEMUCapsTypeFromString(str);
+ if (flag < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("Unknown qemu capabilities flag %s"), str);
+ VIR_FREE(str);
+ goto cleanup;
+ }
+ VIR_FREE(str);
+ virQEMUCapsSet(qemuCaps, flag);
+ }
+ }
+ VIR_FREE(nodes);
+
+ if (virXPathUInt("string(.//version)", ctxt, &qemuCaps->version) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing version in QEMU capabilities cache"));
+ goto cleanup;
+ }
+
+ if (virXPathUInt("string(.//kvmVersion)", ctxt, &qemuCaps->kvmVersion) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing version in QEMU capabilities cache"));
+ goto cleanup;
+ }
+
+ if (!(str = virXPathString("string(.//arch)", ctxt))) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing arch in QEMU capabilities cache"));
+ goto cleanup;
+ }
+ if (!(qemuCaps->arch = virArchFromString(str))) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("unknown arch %s in QEMU capabilities cache"), str);
+ goto cleanup;
+ }
+
+ if ((n = virXPathNodeSet(".//cpu", ctxt, &nodes)) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to parse qemu capabilities cpus"));
+ goto cleanup;
+ }
+ if (n > 0) {
+ qemuCaps->ncpuDefinitions = n;
+ if (VIR_ALLOC_N(qemuCaps->cpuDefinitions,
+ qemuCaps->ncpuDefinitions) < 0)
+ goto cleanup;
+
+ for (i = 0; i < n; i++) {
+ if (!(str = virXMLPropString(nodes[i], "name"))) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing cpu name in QEMU capabilities cache"));
+ goto cleanup;
+ }
+ qemuCaps->cpuDefinitions[i] = str;
+ }
+ }
+ VIR_FREE(nodes);
+
+
+ if ((n = virXPathNodeSet(".//machine", ctxt, &nodes)) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("failed to parse qemu capabilities machines"));
+ goto cleanup;
+ }
+ if (n > 0) {
+ qemuCaps->nmachineTypes = n;
+ if (VIR_ALLOC_N(qemuCaps->machineTypes,
+ qemuCaps->nmachineTypes) < 0 ||
+ VIR_ALLOC_N(qemuCaps->machineAliases,
+ qemuCaps->nmachineTypes) < 0 ||
+ VIR_ALLOC_N(qemuCaps->machineMaxCpus,
+ qemuCaps->nmachineTypes) < 0)
+ goto cleanup;
+
+ for (i = 0; i < n; i++) {
+ if (!(str = virXMLPropString(nodes[i], "name"))) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("missing machine name in QEMU capabilities cache"));
+ goto cleanup;
+ }
+ qemuCaps->machineTypes[i] = str;
+
+ qemuCaps->machineAliases[i] = virXMLPropString(nodes[i], "alias");
+
+ str = virXMLPropString(nodes[i], "maxCpus");
+ if (str &&
+ virStrToLong_ui(str, NULL, 10, &(qemuCaps->machineMaxCpus[i])) < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+ _("malformed machine cpu count in QEMU capabilities cache"));
+ goto cleanup;
+ }
+ }
+ }
+ VIR_FREE(nodes);
+
+ ret = 0;
+ cleanup:
+ VIR_FREE(nodes);
+ xmlXPathFreeContext(ctxt);
+ xmlFreeDoc(doc);
+ return ret;
+}
+
+
+static int
+virQEMUCapsSaveCache(virQEMUCapsPtr qemuCaps, const char *filename)
+{
+ virBuffer buf = VIR_BUFFER_INITIALIZER;
+ const char *xml = NULL;
+ int ret = -1;
+ size_t i;
+ struct utimbuf ut;
+
+ virBufferAddLit(&buf, "<qemuCaps>\n");
+
+ if (qemuCaps->usedQMP)
+ virBufferAddLit(&buf, " <usedQMP/>\n");
+
+ for (i = 0; i < QEMU_CAPS_LAST; i++) {
+ if (virQEMUCapsGet(qemuCaps, i)) {
+ virBufferAsprintf(&buf, " <flag name='%s'/>\n",
+ virQEMUCapsTypeToString(i));
+ }
+ }
+
+ virBufferAsprintf(&buf, " <version>%d</version>\n",
+ qemuCaps->version);
+
+ virBufferAsprintf(&buf, " <kvmVersion>%d</kvmVersion>\n",
+ qemuCaps->kvmVersion);
+
+ virBufferAsprintf(&buf, " <arch>%s</arch>\n",
+ virArchToString(qemuCaps->arch));
+
+ for (i = 0; i < qemuCaps->ncpuDefinitions; i++) {
+ virBufferEscapeString(&buf, " <cpu name='%s'/>\n",
+ qemuCaps->cpuDefinitions[i]);
+ }
+
+ for (i = 0; i < qemuCaps->nmachineTypes; i++) {
+ virBufferEscapeString(&buf, " <machine name='%s'",
+ qemuCaps->machineTypes[i]);
+ if (qemuCaps->machineAliases[i])
+ virBufferEscapeString(&buf, " alias='%s'",
+ qemuCaps->machineAliases[i]);
+ virBufferAsprintf(&buf, " maxCpus='%u'/>\n",
+ qemuCaps->machineMaxCpus[i]);
+ }
+
+ virBufferAddLit(&buf, "</qemuCaps>\n");
+
+ if (virBufferError(&buf))
+ goto cleanup;
+
+ xml = virBufferContentAndReset(&buf);
+
+ if (virFileWriteStr(filename, xml, 0600) < 0) {
+ virReportSystemError(errno,
+ _("Failed to save '%s' for '%s'"),
+ filename, qemuCaps->binary);
+ goto cleanup;
+ }
+
+ ut.actime = qemuCaps->mtime;
+ ut.modtime = qemuCaps->mtime;
+ if (utime(filename, &ut) < 0) {
+ virReportSystemError(errno,
+ _("Failed to set mtime on '%s' for '%s'"),
+ filename, qemuCaps->binary);
+ ignore_value(unlink(filename));
+ goto cleanup;
+ }
+
+ VIR_DEBUG("Saved caps '%s' for '%s' with %lld",
+ filename, qemuCaps->binary, (long long)qemuCaps->mtime);
+
+ ret = 0;
+ cleanup:
+ VIR_FREE(xml);
+ return ret;
+}
+
+static int
+virQEMUCapsRememberCached(virQEMUCapsPtr qemuCaps, const char *cacheDir)
+{
+ char *capsdir = NULL;
+ char *capsfile = NULL;
+ int ret = -1;
+ char *binaryhash = NULL;
+
+ if (virAsprintf(&capsdir, "%s/capabilities", cacheDir) < 0)
+ goto cleanup;
+
+ if (virCryptoHashString(VIR_CRYPTO_HASH_SHA256,
+ qemuCaps->binary,
+ &binaryhash) < 0)
+ goto cleanup;
+
+ if (virAsprintf(&capsfile, "%s/%s.xml", capsdir, binaryhash) < 0)
+ goto cleanup;
+
+ if (virFileMakePath(capsdir) < 0) {
+ virReportSystemError(errno,
+ _("Unable to create directory '%s'"),
+ capsdir);
+ goto cleanup;
+ }
+
+ if (virQEMUCapsSaveCache(qemuCaps, capsfile) < 0)
+ goto cleanup;
+
+ ret = 0;
+ cleanup:
+ VIR_FREE(binaryhash);
+ VIR_FREE(capsfile);
+ VIR_FREE(capsdir);
+ return ret;
+}
+
+
+static void
+virQEMUCapsReset(virQEMUCapsPtr qemuCaps)
+{
+ size_t i;
+
+ virBitmapClearAll(qemuCaps->flags);
+ qemuCaps->version = qemuCaps->kvmVersion = 0;
+ qemuCaps->arch = VIR_ARCH_NONE;
+
+ for (i = 0; i < qemuCaps->ncpuDefinitions; i++) {
+ VIR_FREE(qemuCaps->cpuDefinitions[i]);
+ }
+ VIR_FREE(qemuCaps->cpuDefinitions);
+ qemuCaps->ncpuDefinitions = 0;
+
+ for (i = 0; i < qemuCaps->nmachineTypes; i++) {
+ VIR_FREE(qemuCaps->machineTypes[i]);
+ VIR_FREE(qemuCaps->machineAliases[i]);
+ }
+ VIR_FREE(qemuCaps->machineTypes);
+ VIR_FREE(qemuCaps->machineAliases);
+ qemuCaps->nmachineTypes = 0;
+}
+
+
+static int
+virQEMUCapsInitCached(virQEMUCapsPtr qemuCaps, const char *cacheDir)
+{
+ char *capsdir = NULL;
+ char *capsfile = NULL;
+ int ret = -1;
+ char *binaryhash = NULL;
+ struct stat sb;
+
+ if (virAsprintf(&capsdir, "%s/capabilities", cacheDir) < 0)
+ goto cleanup;
+
+ if (virCryptoHashString(VIR_CRYPTO_HASH_SHA256,
+ qemuCaps->binary,
+ &binaryhash) < 0)
+ goto cleanup;
+
+ if (virAsprintf(&capsfile, "%s/%s.xml", capsdir, binaryhash) < 0)
+ goto cleanup;
+
+ if (virFileMakePath(capsdir) < 0) {
+ virReportSystemError(errno,
+ _("Unable to create directory '%s'"),
+ capsdir);
+ goto cleanup;
+ }
+
+ if (stat(capsfile, &sb) < 0) {
+ if (errno == ENOENT) {
+ VIR_DEBUG("No cached capabilities '%s' for '%s'",
+ capsfile, qemuCaps->binary);
+ ret = 0;
+ goto cleanup;
+ }
+ virReportSystemError(errno,
+ _("Unable to access cache '%s' for '%s'"),
+ capsfile, qemuCaps->binary);
+ goto cleanup;
+ }
+
+ /* Discard if cache is older that QEMU binary */
+ /* XXX must also compare to libvirtd timestamp */
+ if (sb.st_mtime < qemuCaps->mtime) {
+ VIR_DEBUG("Outdated cached capabilities '%s' for '%s' (%lld vs %lld)",
+ capsfile, qemuCaps->binary,
+ (long long)sb.st_mtime, (long long)qemuCaps->mtime);
+ ignore_value(unlink(capsfile));
+ ret = 0;
+ goto cleanup;
+ }
+
+ if (virQEMUCapsLoadCache(qemuCaps, capsfile) < 0) {
+ virErrorPtr err = virGetLastError();
+ VIR_WARN("Failed to load cached caps from '%s' for '%s': %s",
+ capsfile, qemuCaps->binary, err ? NULLSTR(err->message) :
+ _("unknown error"));
+ virResetLastError();
+ ret = 0;
+ virQEMUCapsReset(qemuCaps);
+ goto cleanup;
+ }
+
+ VIR_DEBUG("Loaded '%s' for '%s' mtime %lld usedQMP=%d",
+ capsfile, qemuCaps->binary,
+ (long long)qemuCaps->mtime, qemuCaps->usedQMP);
+
+ ret = 1;
+ cleanup:
+ VIR_FREE(binaryhash);
+ VIR_FREE(capsfile);
+ VIR_FREE(capsdir);
+ return ret;
+}
+
+
#define QEMU_SYSTEM_PREFIX "qemu-system-"
static int
@@ -2779,6 +3161,7 @@ virQEMUCapsLogProbeFailure(const char *binary)
virQEMUCapsPtr virQEMUCapsNewForBinary(const char *binary,
const char *libDir,
+ const char *cacheDir,
uid_t runUid,
gid_t runGid)
{
@@ -2808,15 +3191,23 @@ virQEMUCapsPtr virQEMUCapsNewForBinary(const char *binary,
goto error;
}
- if ((rv = virQEMUCapsInitQMP(qemuCaps, libDir, runUid, runGid)) < 0) {
- virQEMUCapsLogProbeFailure(binary);
+ if ((rv = virQEMUCapsInitCached(qemuCaps, cacheDir)) < 0)
goto error;
- }
- if (!qemuCaps->usedQMP &&
- virQEMUCapsInitHelp(qemuCaps, runUid, runGid) < 0) {
- virQEMUCapsLogProbeFailure(binary);
- goto error;
+ if (rv == 0) {
+ if (virQEMUCapsInitQMP(qemuCaps, libDir, runUid, runGid) < 0) {
+ virQEMUCapsLogProbeFailure(binary);
+ goto error;
+ }
+
+ if (!qemuCaps->usedQMP &&
+ virQEMUCapsInitHelp(qemuCaps, runUid, runGid) < 0) {
+ virQEMUCapsLogProbeFailure(binary);
+ goto error;
+ }
+
+ if (virQEMUCapsRememberCached(qemuCaps, cacheDir) < 0)
+ goto error;
}
return qemuCaps;
@@ -2851,6 +3242,7 @@ virQEMUCapsHashDataFree(void *payload, const void *key ATTRIBUTE_UNUSED)
virQEMUCapsCachePtr
virQEMUCapsCacheNew(const char *libDir,
+ const char *cacheDir,
uid_t runUid,
gid_t runGid)
{
@@ -2870,6 +3262,8 @@ virQEMUCapsCacheNew(const char *libDir,
goto error;
if (VIR_STRDUP(cache->libDir, libDir) < 0)
goto error;
+ if (VIR_STRDUP(cache->cacheDir, cacheDir) < 0)
+ goto error;
cache->runUid = runUid;
cache->runGid = runGid;
@@ -2899,6 +3293,7 @@ virQEMUCapsCacheLookup(virQEMUCapsCachePtr cache, const char *binary)
VIR_DEBUG("Creating capabilities for %s",
binary);
ret = virQEMUCapsNewForBinary(binary, cache->libDir,
+ cache->cacheDir,
cache->runUid, cache->runGid);
if (ret) {
VIR_DEBUG("Caching capabilities %p for %s",
@@ -2938,6 +3333,7 @@ virQEMUCapsCacheFree(virQEMUCapsCachePtr cache)
return;
VIR_FREE(cache->libDir);
+ VIR_FREE(cache->cacheDir);
virHashFree(cache->binaries);
virMutexDestroy(&cache->lock);
VIR_FREE(cache);
diff --git a/src/qemu/qemu_capabilities.h b/src/qemu/qemu_capabilities.h
index b5445e7..a9082d5 100644
--- a/src/qemu/qemu_capabilities.h
+++ b/src/qemu/qemu_capabilities.h
@@ -218,6 +218,7 @@ virQEMUCapsPtr virQEMUCapsNew(void);
virQEMUCapsPtr virQEMUCapsNewCopy(virQEMUCapsPtr qemuCaps);
virQEMUCapsPtr virQEMUCapsNewForBinary(const char *binary,
const char *libDir,
+ const char *cacheDir,
uid_t runUid,
gid_t runGid);
@@ -262,6 +263,7 @@ bool virQEMUCapsIsValid(virQEMUCapsPtr qemuCaps);
virQEMUCapsCachePtr virQEMUCapsCacheNew(const char *libDir,
+ const char *cacheDir,
uid_t uid, gid_t gid);
virQEMUCapsPtr virQEMUCapsCacheLookup(virQEMUCapsCachePtr cache,
const char *binary);
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 9aad2dc..51c9ed0 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -744,6 +744,7 @@ qemuStateInitialize(bool privileged,
}
qemu_driver->qemuCapsCache = virQEMUCapsCacheNew(cfg->libDir,
+ cfg->cacheDir,
run_uid,
run_gid);
if (!qemu_driver->qemuCapsCache)
--
1.8.5.3
--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list