Intend to reduce the redundant code,use virNumaSetupMemoryPolicy to replace virLXCControllerSetupNUMAPolicy and qemuProcessInitNumaMemoryPolicy. This patch also moves the numa related codes to the file virnuma.c and virnuma.h Signed-off-by: Gao feng <gaofeng@xxxxxxxxxxxxxx> --- src/conf/domain_conf.c | 17 ++----- src/conf/domain_conf.h | 17 +------ src/libvirt_private.syms | 5 +- src/lxc/lxc_controller.c | 114 +----------------------------------------- src/qemu/qemu_cgroup.c | 4 +- src/qemu/qemu_process.c | 121 +-------------------------------------------- src/util/virnuma.c | 126 +++++++++++++++++++++++++++++++++++++++++++++++ src/util/virnuma.h | 30 +++++++++++ 8 files changed, 168 insertions(+), 266 deletions(-) diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c index 3278e9c..72214b1 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c @@ -690,11 +690,6 @@ VIR_ENUM_IMPL(virDomainTimerMode, VIR_DOMAIN_TIMER_MODE_LAST, "paravirt", "smpsafe"); -VIR_ENUM_IMPL(virDomainNumatuneMemMode, VIR_DOMAIN_NUMATUNE_MEM_LAST, - "strict", - "preferred", - "interleave"); - VIR_ENUM_IMPL(virDomainStartupPolicy, VIR_DOMAIN_STARTUP_POLICY_LAST, "default", "mandatory", @@ -709,12 +704,6 @@ VIR_ENUM_IMPL(virDomainDiskTray, VIR_DOMAIN_DISK_TRAY_LAST, "closed", "open"); -VIR_ENUM_IMPL(virDomainNumatuneMemPlacementMode, - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_LAST, - "default", - "static", - "auto"); - VIR_ENUM_IMPL(virDomainRNGModel, VIR_DOMAIN_RNG_MODEL_LAST, "virtio"); @@ -9851,7 +9840,7 @@ virDomainDefParseXML(virCapsPtr caps, int placement_mode = 0; if (placement) { if ((placement_mode = - virDomainNumatuneMemPlacementModeTypeFromString(placement)) < 0) { + virNumatuneMemPlacementModeTypeFromString(placement)) < 0) { virReportError(VIR_ERR_XML_ERROR, _("Unsupported memory placement " "mode '%s'"), placement); @@ -9906,7 +9895,7 @@ virDomainDefParseXML(virCapsPtr caps, * and 'placement' of <vcpu> is 'auto'. */ if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) { - def->numatune.memory.placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO; + def->numatune.memory.placement_mode = VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO; def->numatune.memory.mode = VIR_DOMAIN_NUMATUNE_MEM_STRICT; } } @@ -14827,7 +14816,7 @@ virDomainDefFormatInternal(virDomainDefPtr def, virBufferAsprintf(buf, "nodeset='%s'/>\n", nodemask); VIR_FREE(nodemask); } else if (def->numatune.memory.placement_mode) { - placement = virDomainNumatuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode); + placement = virNumatuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode); virBufferAsprintf(buf, "placement='%s'/>\n", placement); } virBufferAddLit(buf, " </numatune>\n"); diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h index 96f11ba..98c4745 100644 --- a/src/conf/domain_conf.h +++ b/src/conf/domain_conf.h @@ -47,6 +47,7 @@ # include "device_conf.h" # include "virbitmap.h" # include "virstoragefile.h" +# include "virnuma.h" /* forward declarations of all device types, required by * virDomainDeviceDef @@ -1700,18 +1701,6 @@ virDomainVcpuPinDefPtr virDomainVcpuPinFindByVcpu(virDomainVcpuPinDefPtr *def, int nvcpupin, int vcpu); -typedef struct _virDomainNumatuneDef virDomainNumatuneDef; -typedef virDomainNumatuneDef *virDomainNumatuneDefPtr; -struct _virDomainNumatuneDef { - struct { - virBitmapPtr nodemask; - int mode; - int placement_mode; /* enum virDomainNumatuneMemPlacementMode */ - } memory; - - /* Future NUMA tuning related stuff should go here. */ -}; - typedef struct _virBlkioDeviceWeight virBlkioDeviceWeight; typedef virBlkioDeviceWeight *virBlkioDeviceWeightPtr; struct _virBlkioDeviceWeight { @@ -1801,7 +1790,7 @@ struct _virDomainDef { virDomainVcpuPinDefPtr emulatorpin; } cputune; - virDomainNumatuneDef numatune; + virNumatuneDef numatune; /* These 3 are based on virDomainLifeCycleAction enum flags */ int onReboot; @@ -2396,8 +2385,6 @@ VIR_ENUM_DECL(virDomainGraphicsSpicePlaybackCompression) VIR_ENUM_DECL(virDomainGraphicsSpiceStreamingMode) VIR_ENUM_DECL(virDomainGraphicsSpiceClipboardCopypaste) VIR_ENUM_DECL(virDomainGraphicsSpiceMouseMode) -VIR_ENUM_DECL(virDomainNumatuneMemMode) -VIR_ENUM_DECL(virDomainNumatuneMemPlacementMode) VIR_ENUM_DECL(virDomainHyperv) VIR_ENUM_DECL(virDomainRNGModel) VIR_ENUM_DECL(virDomainRNGBackend) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 1374470..a4e6801 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -254,8 +254,8 @@ virDomainNostateReasonTypeFromString; virDomainNostateReasonTypeToString; virDomainNumatuneMemModeTypeFromString; virDomainNumatuneMemModeTypeToString; -virDomainNumatuneMemPlacementModeTypeFromString; -virDomainNumatuneMemPlacementModeTypeToString; +virNumatuneMemPlacementModeTypeFromString; +virNumatuneMemPlacementModeTypeToString; virDomainObjAssignDef; virDomainObjCopyPersistentDef; virDomainObjGetPersistentDef; @@ -1547,6 +1547,7 @@ virNodeSuspendGetTargetMask; # util/virnuma.h virNumaGetAutoPlacementAdvice; +virNumaSetupMemoryPolicy; # util/virobject.h virClassForObject; diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c index bb4cd16..6c5b8c8 100644 --- a/src/lxc/lxc_controller.c +++ b/src/lxc/lxc_controller.c @@ -46,11 +46,6 @@ # include <cap-ng.h> #endif -#if WITH_NUMACTL -# define NUMA_VERSION1_COMPATIBILITY 1 -# include <numa.h> -#endif - #include "virerror.h" #include "virlog.h" #include "virutil.h" @@ -479,113 +474,6 @@ cleanup: return ret; } -#if WITH_NUMACTL -static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl, - virBitmapPtr nodemask) -{ - nodemask_t mask; - int mode = -1; - int node = -1; - int ret = -1; - int i = 0; - int maxnode = 0; - bool warned = false; - virDomainNumatuneDef numatune = ctrl->def->numatune; - virBitmapPtr tmp_nodemask = NULL; - - if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) { - if (!numatune.memory.nodemask) - return 0; - VIR_DEBUG("Set NUMA memory policy with specified nodeset"); - tmp_nodemask = numatune.memory.nodemask; - } else if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) { - VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad"); - tmp_nodemask = nodemask; - } else { - return 0; - } - - VIR_DEBUG("Setting NUMA memory policy"); - - if (numa_available() < 0) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - "%s", _("Host kernel is not aware of NUMA.")); - return -1; - } - - maxnode = numa_max_node() + 1; - - /* Convert nodemask to NUMA bitmask. */ - nodemask_zero(&mask); - i = -1; - while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) { - if (i > NUMA_NUM_NODES) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("Host cannot support NUMA node %d"), i); - return -1; - } - if (i > maxnode && !warned) { - VIR_WARN("nodeset is out of range, there is only %d NUMA " - "nodes on host", maxnode); - warned = true; - } - nodemask_set(&mask, i); - } - - mode = ctrl->def->numatune.memory.mode; - - if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { - numa_set_bind_policy(1); - numa_set_membind(&mask); - numa_set_bind_policy(0); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { - int nnodes = 0; - for (i = 0; i < NUMA_NUM_NODES; i++) { - if (nodemask_isset(&mask, i)) { - node = i; - nnodes++; - } - } - - if (nnodes != 1) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - "%s", _("NUMA memory tuning in 'preferred' mode " - "only supports single node")); - goto cleanup; - } - - numa_set_bind_policy(0); - numa_set_preferred(node); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { - numa_set_interleave_mask(&mask); - } else { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, - _("Unable to set NUMA policy %s"), - virDomainNumatuneMemModeTypeToString(mode)); - goto cleanup; - } - - ret = 0; - -cleanup: - return ret; -} -#else -static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl, - virBitmapPtr nodemask ATTRIBUTE_UNUSED) -{ - if (ctrl->def->numatune.memory.nodemask) { - virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", - _("NUMA policy is not available on this platform")); - return -1; - } - - return 0; -} -#endif - /* * To be run while still single threaded @@ -684,7 +572,7 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl) int ret = -1; if (virLXCControllerGetNumadAdvice(ctrl, &nodemask) < 0 || - virLXCControllerSetupNUMAPolicy(ctrl, nodemask) < 0) + virNumaSetupMemoryPolicy(ctrl->def->numatune, nodemask) < 0) goto cleanup; if (virLXCControllerSetupCpuAffinity(ctrl) < 0) diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index 9d6e88b..f2c5df8 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -423,12 +423,12 @@ int qemuSetupCgroup(virQEMUDriverPtr driver, if ((vm->def->numatune.memory.nodemask || (vm->def->numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) && + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) && vm->def->numatune.memory.mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT && qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) { char *mask = NULL; if (vm->def->numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) mask = virBitmapFormat(nodemask); else mask = virBitmapFormat(vm->def->numatune.memory.nodemask); diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 96ba70a..f12e54a 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -45,11 +45,6 @@ #include "qemu_bridge_filter.h" #include "qemu_migration.h" -#if WITH_NUMACTL -# define NUMA_VERSION1_COMPATIBILITY 1 -# include <numa.h> -#endif - #include "datatypes.h" #include "virlog.h" #include "virerror.h" @@ -1791,120 +1786,6 @@ qemuProcessDetectVcpuPIDs(virQEMUDriverPtr driver, } -/* - * Set NUMA memory policy for qemu process, to be run between - * fork/exec of QEMU only. - */ -#if WITH_NUMACTL -static int -qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm, - virBitmapPtr nodemask) -{ - nodemask_t mask; - int mode = -1; - int node = -1; - int ret = -1; - int i = 0; - int maxnode = 0; - bool warned = false; - virDomainNumatuneDef numatune = vm->def->numatune; - virBitmapPtr tmp_nodemask = NULL; - - if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) { - if (!numatune.memory.nodemask) - return 0; - VIR_DEBUG("Set NUMA memory policy with specified nodeset"); - tmp_nodemask = numatune.memory.nodemask; - } else if (numatune.memory.placement_mode == - VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) { - VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad"); - tmp_nodemask = nodemask; - } else { - return 0; - } - - if (numa_available() < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("Host kernel is not aware of NUMA.")); - return -1; - } - - maxnode = numa_max_node() + 1; - /* Convert nodemask to NUMA bitmask. */ - nodemask_zero(&mask); - i = -1; - while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) { - if (i > NUMA_NUM_NODES) { - virReportError(VIR_ERR_INTERNAL_ERROR, - _("Host cannot support NUMA node %d"), i); - return -1; - } - if (i > maxnode && !warned) { - VIR_WARN("nodeset is out of range, there is only %d NUMA " - "nodes on host", maxnode); - warned = true; - } - nodemask_set(&mask, i); - } - - mode = numatune.memory.mode; - - if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { - numa_set_bind_policy(1); - numa_set_membind(&mask); - numa_set_bind_policy(0); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { - int nnodes = 0; - for (i = 0; i < NUMA_NUM_NODES; i++) { - if (nodemask_isset(&mask, i)) { - node = i; - nnodes++; - } - } - - if (nnodes != 1) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("NUMA memory tuning in 'preferred' mode " - "only supports single node")); - goto cleanup; - } - - numa_set_bind_policy(0); - numa_set_preferred(node); - } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { - numa_set_interleave_mask(&mask); - } else { - /* XXX: Shouldn't go here, as we already do checking when - * parsing domain XML. - */ - virReportError(VIR_ERR_XML_ERROR, - "%s", _("Invalid mode for memory NUMA tuning.")); - goto cleanup; - } - - ret = 0; - -cleanup: - return ret; -} -#else -static int -qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm, - virBitmapPtr nodemask ATTRIBUTE_UNUSED) -{ - if (vm->def->numatune.memory.nodemask) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("libvirt is compiled without NUMA tuning support")); - - return -1; - } - - return 0; -} -#endif - - /* Helper to prepare cpumap for affinity setting, convert * NUMA nodeset into cpuset if @nodemask is not NULL, otherwise * just return a new allocated bitmap. @@ -2654,7 +2535,7 @@ static int qemuProcessHook(void *data) qemuProcessInitCpuAffinity(h->driver, h->vm, h->nodemask) < 0) goto cleanup; - if (qemuProcessInitNumaMemoryPolicy(h->vm, h->nodemask) < 0) + if (virNumaSetupMemoryPolicy(h->vm->def->numatune, h->nodemask) < 0) goto cleanup; ret = 0; diff --git a/src/util/virnuma.c b/src/util/virnuma.c index f6a6eb2..e60df42 100644 --- a/src/util/virnuma.c +++ b/src/util/virnuma.c @@ -21,12 +21,30 @@ #include <config.h> +#if WITH_NUMACTL +# define NUMA_VERSION1_COMPATIBILITY 1 +# include <numa.h> +#endif + #include "virnuma.h" #include "vircommand.h" #include "virerror.h" +#include "virlog.h" #define VIR_FROM_THIS VIR_FROM_NONE +VIR_ENUM_IMPL(virDomainNumatuneMemMode, + VIR_DOMAIN_NUMATUNE_MEM_LAST, + "strict", + "preferred", + "interleave"); + +VIR_ENUM_IMPL(virNumatuneMemPlacementMode, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_LAST, + "default", + "static", + "auto"); + #if HAVE_NUMAD char * virNumaGetAutoPlacementAdvice(unsigned short vcpus, @@ -59,3 +77,111 @@ virNumaGetAutoPlacementAdvice(unsigned short vcpus ATTRIBUTE_UNUSED, return NULL; } #endif + +#if WITH_NUMACTL +int +virNumaSetupMemoryPolicy(virNumatuneDef numatune, + virBitmapPtr nodemask) +{ + nodemask_t mask; + int mode = -1; + int node = -1; + int ret = -1; + int i = 0; + int maxnode = 0; + bool warned = false; + virBitmapPtr tmp_nodemask = NULL; + + if (numatune.memory.placement_mode == + VIR_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) { + if (!numatune.memory.nodemask) + return 0; + VIR_DEBUG("Set NUMA memory policy with specified nodeset"); + tmp_nodemask = numatune.memory.nodemask; + } else if (numatune.memory.placement_mode == + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) { + VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad"); + tmp_nodemask = nodemask; + } else { + return 0; + } + + if (numa_available() < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Host kernel is not aware of NUMA.")); + return -1; + } + + maxnode = numa_max_node() + 1; + /* Convert nodemask to NUMA bitmask. */ + nodemask_zero(&mask); + i = -1; + while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) { + if (i > NUMA_NUM_NODES) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Host cannot support NUMA node %d"), i); + return -1; + } + if (i > maxnode && !warned) { + VIR_WARN("nodeset is out of range, there is only %d NUMA " + "nodes on host", maxnode); + warned = true; + } + nodemask_set(&mask, i); + } + + mode = numatune.memory.mode; + + if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) { + numa_set_bind_policy(1); + numa_set_membind(&mask); + numa_set_bind_policy(0); + } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) { + int nnodes = 0; + for (i = 0; i < NUMA_NUM_NODES; i++) { + if (nodemask_isset(&mask, i)) { + node = i; + nnodes++; + } + } + + if (nnodes != 1) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("NUMA memory tuning in 'preferred' mode " + "only supports single node")); + goto cleanup; + } + + numa_set_bind_policy(0); + numa_set_preferred(node); + } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) { + numa_set_interleave_mask(&mask); + } else { + /* XXX: Shouldn't go here, as we already do checking when + * parsing domain XML. + */ + virReportError(VIR_ERR_XML_ERROR, + "%s", _("Invalid mode for memory NUMA tuning.")); + goto cleanup; + } + + ret = 0; + +cleanup: + return ret; +} +#else +int +virNumaSetupMemoryPolicy(virNumatuneDef numatune, + virBitmapPtr nodemask ATTRIBUTE_UNUSED) +{ + if (numatune.memory.nodemask) { + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("libvirt is compiled without NUMA tuning support")); + + return -1; + } + + return 0; +} +#endif diff --git a/src/util/virnuma.h b/src/util/virnuma.h index d3d7d3e..ef201b9 100644 --- a/src/util/virnuma.h +++ b/src/util/virnuma.h @@ -22,7 +22,37 @@ #ifndef __VIR_NUMA_H__ # define __VIR_NUMA_H__ +# include "internal.h" +# include "virbitmap.h" +# include "virutil.h" + +enum virNumatuneMemPlacementMode { + VIR_NUMATUNE_MEM_PLACEMENT_MODE_DEFAULT = 0, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_STATIC, + VIR_NUMATUNE_MEM_PLACEMENT_MODE_AUTO, + + VIR_NUMATUNE_MEM_PLACEMENT_MODE_LAST +}; + +VIR_ENUM_DECL(virNumatuneMemPlacementMode) + +VIR_ENUM_DECL(virDomainNumatuneMemMode) + +typedef struct _virNumatuneDef virNumatuneDef; +typedef virNumatuneDef *virNumatuneDefPtr; +struct _virNumatuneDef { + struct { + virBitmapPtr nodemask; + int mode; + int placement_mode; /* enum virNumatuneMemPlacementMode */ + } memory; + + /* Future NUMA tuning related stuff should go here. */ +}; + char *virNumaGetAutoPlacementAdvice(unsigned short vcups, unsigned long long balloon); +int virNumaSetupMemoryPolicy(virNumatuneDef numatune, + virBitmapPtr nodemask); #endif /* __VIR_NUMA_H__ */ -- 1.7.11.7 -- libvir-list mailing list libvir-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/libvir-list