Hi Eli Qiao,
On Wed, Jan 18, 2017 at 03:26:19PM +0800, Eli Qiao wrote:
> This patch adds some utils struct and functions to expose resctrl
> information.
>
> virResCtrlAvailable: If resctrl interface exist on host
> virResCtrlGet: get specify type resource contral information
> virResCtrlInit: initialize resctrl struct from the host's sys fs.
> ResCtrlAll[]: an array to maintain resource contral information.
>
> Signed-off-by: Eli Qiao <liyong.qiao@xxxxxxxxx>
> ---
> include/libvirt/virterror.h | 1 +
> src/Makefile.am | 1 +
> src/libvirt_private.syms | 4 +
> src/util/virerror.c | 1 +
> src/util/virresctrl.c | 500 ++++++++++++++++++++++++++++++++++++++++++++
> src/util/virresctrl.h | 121 +++++++++++
> tests/Makefile.am | 4 +
> 7 files changed, 632 insertions(+)
> create mode 100644 src/util/virresctrl.c
> create mode 100644 src/util/virresctrl.h
>
> diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h
> index 2efee8f..3dd2d08 100644
> --- a/include/libvirt/virterror.h
> +++ b/include/libvirt/virterror.h
> @@ -132,6 +132,7 @@ typedef enum {
>
> VIR_FROM_PERF = 65, /* Error from perf */
> VIR_FROM_LIBSSH = 66, /* Error from libssh connection transport */
> + VIR_FROM_RESCTRL = 67, /* Error from resource control */
>
> # ifdef VIR_ENUM_SENTINELS
> VIR_ERR_DOMAIN_LAST
> diff --git a/src/Makefile.am b/src/Makefile.am
> index d84c984..4026f8d 100644
> --- a/src/Makefile.am
> +++ b/src/Makefile.am
> @@ -162,6 +162,7 @@ UTIL_SOURCES = \
> util/virprocess.c util/virprocess.h \
> util/virqemu.c util/virqemu.h \
> util/virrandom.h util/virrandom.c \
> + util/virresctrl.h util/virresctrl.c \
> util/virrotatingfile.h util/virrotatingfile.c \
> util/virscsi.c util/virscsi.h \
> util/virscsivhost.c util/virscsivhost.h \
> diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
> index 2d23e46..fe1334d 100644
> --- a/src/libvirt_private.syms
> +++ b/src/libvirt_private.syms
> @@ -2295,6 +2295,10 @@ virRandomGenerateWWN;
> virRandomInt;
>
>
> +# util/virresctrl.h
> +virResCtrlAvailable;
> +virResCtrlInit;
> +
> # util/virrotatingfile.h
> virRotatingFileReaderConsume;
> virRotatingFileReaderFree;
> diff --git a/src/util/virerror.c b/src/util/virerror.c
> index ef17fb5..93dfd4f 100644
> --- a/src/util/virerror.c
> +++ b/src/util/virerror.c
> @@ -139,6 +139,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST,
>
> "Perf", /* 65 */
> "Libssh transport layer",
> + "Rescouce Control",
Typo.
> diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
> new file mode 100644
> index 0000000..bcb47f1
> --- /dev/null
> +++ b/src/util/virresctrl.c
> @@ -0,0 +1,500 @@
> +/*
> + * virresctrl.c: methods for managing resource contral
control
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library. If not, see
> + * <http://www.gnu.org/licenses/>.
> + *
> + * Authors:
> + * Eli Qiao <liyong.qiao@xxxxxxxxx>
> + */
> +#include <config.h>
> +
> +#include <sys/ioctl.h>
> +#if defined HAVE_SYS_SYSCALL_H
> +# include <sys/syscall.h>
> +#endif
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <fcntl.h>
> +
> +#include "virresctrl.h"
> +#include "viralloc.h"
> +#include "virerror.h"
> +#include "virfile.h"
> +#include "virhostcpu.h"
> +#include "virlog.h"
> +#include "virstring.h"
> +#include "nodeinfo.h"
> +
> +VIR_LOG_INIT("util.resctrl");
> +
> +#define VIR_FROM_THIS VIR_FROM_RESCTRL
> +
> +#define CONSTRUCT_RESCTRL_PATH(domain_name, item_name) \
> +do { \
> + if (NULL == domain_name) { \
> + if (asprintf(&path, "%s/%s", RESCTRL_DIR, item_name) < 0)\
> + return -1; \
> + } \
> + else { \
> + if (asprintf(&path, "%s/%s/%s", RESCTRL_DIR, \
> + domain_name, \
> + item_name) < 0) \
> + return -1; \
> + } \
> +} while(0)
> +
> +static virResCtrl ResCtrlAll[] = {
> + {
> + .name = "L3",
> + .domains = NULL,
> + .cache_level = "l3",
> + },
> + {
> + .name = "L3DATA",
> + .domains = NULL,
> + .cache_level = "l3",
> + },
> + {
> + .name = "L3CODE",
> + .domains = NULL,
> + .cache_level = "l3",
> + },
> + {
> + .name = "L2",
> + .domains = NULL,
> + .cache_level = "l2",
> + },
> +};
> +
> +/* Return pointer of and ncount of schemata*/
> +static virResSchemataPtr virParseSchemata(const char* schemata, int *ncount)
> +{
> + char type[MAX_SCHEMATA_LEN];
> + const char *p, *q;
> + int pos;
> + int ischemata;
> + virResSchemataPtr tmpschemata, pschemata;
> +
> + unsigned int socket_no = 0;
> + p = q = schemata;
> + pos = strchr(schemata, ':') - p;
> +
> + if(virStrncpy(type, schemata, pos, strlen(schemata)) == NULL) {
> + return NULL;
> + }
> +
> + *ncount = 1;
> + while((q = strchr(p, ';')) != 0) {
> + p = q + 1;
> + (*ncount) ++;
> + }
> +
> + /* allocat an arrry to keep schemata */
typo
> + if(VIR_ALLOC_N_QUIET(tmpschemata, *ncount) < 0) {
> + return NULL;
> + }
> +
> + pschemata = tmpschemata;
> +
> + p = q = schemata + pos + 1;
> +
> + char *tmp;
> +
> + while(*p != '\0'){
> + if (*p == '='){
> + q = p + 1;
> +
> + if (VIR_STRDUP(tmpschemata->name, type) < 0)
> + goto cleanup;
> +
> + tmpschemata->socket_no = socket_no++;
Isnt this incorrect if there are two lines (one for data and one for
code) in the schemata file?
> + while(*p != ';' && *p != '\0') p++;
> +
> + if (VIR_STRNDUP(tmp, q, p-q) < 0)
> + goto cleanup;
> +
> + if (virStrToLong_i(tmp, NULL, 16, &ischemata) < 0)
> + goto cleanup;
> +
> + VIR_FREE(tmp);
> + tmpschemata->schemata = ischemata;
> + tmpschemata ++;
> + }
> + p++;
> + }
> +
> + return pschemata;
> +
> +cleanup:
> + VIR_FREE(pschemata);
> + return NULL;
> +}
> +
> +static int virResCtrlGetStr(const char *domain_name, const char *item_name, char **ret)
> +{
> + char *path;
> + int rc = 0;
> +
> + CONSTRUCT_RESCTRL_PATH(domain_name, item_name);
> +
> + if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) {
> + rc = -1;
> + goto cleanup;
> + }
> +
> +cleanup:
> + VIR_FREE(path);
> + return rc;
> +}
> +
> +static int virResCtrlGetTasks(const char *domain_name, char **pids)
> +{
> + return virResCtrlGetStr(domain_name, "tasks", pids);
> +}
> +
> +static int virResCtrlGetSchemata(const int type, const char *name, char **schemata)
> +{
> + int rc;
> + char *tmp, *end;
> + char *buf;
> +
> + if ((rc = virResCtrlGetStr(name, "schemata", &buf)) < 0)
> + return rc;
> +
> + tmp = strstr(buf, ResCtrlAll[type].name);
> + end = strchr(tmp, '\n');
> + *end = '\0';
> + if(VIR_STRDUP(*schemata, tmp) < 0)
> + rc = -1;
> +
> + VIR_FREE(buf);
> + return rc;
> +}
> +
> +static int virResCtrlGetInfoStr(const int type, const char *item, char **str)
> +{
> + int ret = 0;
> + char *tmp;
> + char *path;
> +
> + if (asprintf(&path, "%s/%s/%s", RESCTRL_INFO_DIR, ResCtrlAll[type].name, item) < 0)
> + return -1;
> + if (virFileReadAll(path, 10, str) < 0) {
> + ret = -1;
> + goto cleanup;
> + }
> +
> + if ((tmp = strchr(*str, '\n'))) {
> + *tmp = '\0';
> + }
> +
> +cleanup:
> + VIR_FREE(path);
> + return ret;
> +}
> +
> +static virResDomainPtr virResCtrlGetAllDomains(int type, int *len)
> +{
> + struct dirent *ent;
> + DIR *dp = NULL;
> + int direrr;
> + char *schematas, *tasks;
> +
> + *len = 0;
> + virResDomainPtr header, tmp, tmp_pre;
> + header = tmp = tmp_pre = NULL;
> + if (virDirOpenQuiet(&dp, RESCTRL_DIR) < 0) {
> + if (errno == ENOENT)
> + return NULL;
> + VIR_ERROR(_("Unable to open %s (%d)"), RESCTRL_DIR, errno);
> + goto cleanup;
> + }
> +
> + /* read default domain */
> + if(VIR_ALLOC(header) < 0)
> + return NULL;
> + header->pre = NULL;
> + header->next = NULL;
> + if(VIR_STRDUP(header->name, "default") < 0)
> + goto cleanup;
> +
> + if (virResCtrlGetSchemata(type, NULL, &schematas) < 0) {
> + goto cleanup;
> + }
> + header->schematas = virParseSchemata(schematas, &(header->n_sockets));
> + VIR_FREE(schematas);
> + *len = 1;
> +
> + while ((direrr = virDirRead(dp, &ent, NULL)) > 0) {
> + if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info"))
> + continue;
> +
> + if(virResCtrlGetTasks(ent->d_name, &tasks) < 0)
> + continue;
> +
> + if(VIR_ALLOC(tmp) < 0)
> + return NULL;
> +
> + tmp->next = NULL;
> +
> + if(header->next == NULL)
> + header->next = tmp;
> +
> + if(tmp_pre == NULL)
> + tmp->pre = header;
> + else {
> + tmp->pre = tmp_pre;
> + tmp_pre->next = tmp;
> + }
> +
> + if(VIR_STRDUP(tmp->name, ent->d_name) < 0)
> + goto cleanup;
> +
> + tmp_pre = tmp;
> + if (virResCtrlGetSchemata(type, tmp->name, &schematas) < 0) {
> + goto cleanup;
> + }
> +
> + (*len) ++;
> + tmp->schematas = virParseSchemata(schematas, &(tmp->n_sockets));
> + tmp->tasks = tasks;
> + VIR_FREE(schematas);
> + }
> + return header;
> +
> +cleanup:
> +
> + VIR_DIR_CLOSE(dp);
> + return NULL;
> +}
> +
> +
> +static int virResCtrlGetCPUValue(const char* path, char** value)
> +{
> + int ret = -1;
> + char* tmp;
> +
> + if(virFileReadAll(path, 10, value) < 0) {
> + goto cleanup;
> + }
> + if ((tmp = strchr(*value, '\n'))) {
> + *tmp = '\0';
> + }
> + ret = 0;
> +cleanup:
> + return ret;
> +}
> +
> +static int virResctrlGetCPUSocketID(const size_t cpu, int* socket_id)
> +{
> + int ret = -1;
> + char* physical_package_path = NULL;
> + char* physical_package = NULL;
> + if (virAsprintf(&physical_package_path,
> + "%s/cpu/cpu%zu/topology/physical_package_id",
> + SYSFS_SYSTEM_PATH, cpu) < 0) {
> + return -1;
> + }
> +
> + if(virResCtrlGetCPUValue(physical_package_path,
> + &physical_package) < 0)
> + goto cleanup;
> +
> + if (virStrToLong_i(physical_package, NULL, 0, socket_id) < 0)
> + goto cleanup;
> +
> + ret = 0;
> +cleanup:
> + VIR_FREE(physical_package);
> + VIR_FREE(physical_package_path);
> + return ret;
> +}
> +
> +static int virResCtrlGetCPUCache(const size_t cpu, int type, int *cache)
> +{
> + int ret = -1;
> + char* cache_dir = NULL;
> + char* cache_str = NULL;
> + char* tmp;
> + int carry = -1;
> +
> + if (virAsprintf(&cache_dir,
> + "%s/cpu/cpu%zu/cache/index%d/size",
> + SYSFS_SYSTEM_PATH, cpu, type) < 0)
> + return -1;
> +
> + if(virResCtrlGetCPUValue(cache_dir, &cache_str) < 0)
> + goto cleanup;
> +
> + tmp = cache_str;
> +
> + while (*tmp != '\0')
> + tmp++;
> + if (*(tmp - 1) == 'K') {
> + *(tmp - 1) = '\0';
> + carry = 1;
> + }
> + else if (*(tmp - 1) == 'M') {
> + *(tmp - 1) = '\0';
> + carry = 1024;
> + }
> +
> + if (virStrToLong_i(cache_str, NULL, 0, cache) < 0)
> + goto cleanup;
> +
> + *cache = (*cache) * carry;
> +
> + if (*cache < 0)
> + goto cleanup;
> +
> + ret = 0;
> +cleanup:
> + VIR_FREE(cache_dir);
> + VIR_FREE(cache_str);
> + return ret;
> +}
> +
> +/*
> + * Fill cache informations for specify cache type
> +*/
> +static int virResCtrlParseCPUCache(int type)
> +{
> + int index = -1;
> + int npresent_cpus;
> +
> + if ((npresent_cpus = virHostCPUGetCount()) < 0)
> + return -1;
> +
> + if (type == RDT_RESOURCE_L3
> + || type == RDT_RESOURCE_L3DATA
> + || type == RDT_RESOURCE_L3CODE)
> + index = 3;
> + else if (type == RDT_RESOURCE_L2) {
> + index = 2;
> + }
> +
> + if (index == -1)
> + return -1;
> +
> + for(size_t i = 0; i < npresent_cpus ; i ++) {
> + int s_id;
> + int cache_size;
> +
> + if (virResctrlGetCPUSocketID(i, &s_id) < 0) {
> + return -1;
> + }
> + if (ResCtrlAll[type].cpu_mask[s_id] == NULL) {
> + if (!(ResCtrlAll[type].cpu_mask[s_id] = virBitmapNew(npresent_cpus)))
> + return -1;
> + }
> +
> + ignore_value(virBitmapSetBit(ResCtrlAll[type].cpu_mask[s_id], i));
> +
> + if (ResCtrlAll[type].cache_size[s_id] == 0) {
> + if (virResCtrlGetCPUCache(i, index, &cache_size) < 0) {
> + return -1;
> + }
> + ResCtrlAll[type].cache_size[s_id] = cache_size;
> + ResCtrlAll[type].cache_min[s_id] = cache_size / ResCtrlAll[type].cbm_len;
> + }
> + }
> + return 0;
> +}
> +
> +static int virResCtrlGetConfig(int type)
> +{
> + int ret;
> + int i;
> + char *str;
> +
> + /* Read min_cbm_bits from resctrl.
> + eg: /sys/fs/resctrl/info/L3/num_closids
> + */
> + if ((ret = virResCtrlGetInfoStr(type, "num_closids", &str)) < 0) {
> + return ret;
> + }
> + if (virStrToLong_i(str, NULL, 10, &ResCtrlAll[type].num_closid) < 0) {
> + return -1;
> + }
> + VIR_FREE(str);
> +
> + /* Read min_cbm_bits from resctrl.
> + eg: /sys/fs/resctrl/info/L3/cbm_mask
> + */
> + if ((ret = virResCtrlGetInfoStr(type, "min_cbm_bits", &str)) < 0) {
> + return ret;
> + }
> + if (virStrToLong_i(str, NULL, 10, &ResCtrlAll[type].min_cbm_bits) < 0) {
> + return -1;
> + }
> + VIR_FREE(str);
> +
> + /* Read cbm_mask string from resctrl.
> + eg: /sys/fs/resctrl/info/L3/cbm_mask
> + */
> + if ((ret = virResCtrlGetInfoStr(type, "cbm_mask", &str)) < 0) {
> + return ret;
> + }
> +
> + /* Calculate cbm length from the default cbm_mask. */
> + ResCtrlAll[type].cbm_len = strlen(str) * 4;
> + VIR_FREE(str);
> +
> + /* Get all resctrl comains from /sys/fs/resctrl */
> + ResCtrlAll[type].domains = virResCtrlGetAllDomains(type, &ResCtrlAll[type].num_domains);
> + ResCtrlAll[type].num_sockets = ResCtrlAll[type].domains->n_sockets;
> +
> + /* Get cache_left information */
> + for(i = 0; i < ResCtrlAll[type].num_sockets; i++) {
> + ResCtrlAll[type].cpu_mask[i] = NULL;
> + }
> +
> + if((ret = virResCtrlParseCPUCache(type)) < 0)
> + return ret;
> +
> + ResCtrlAll[type].enabled = true;
> +
> + return ret;
> +}
> +
> +int virResCtrlInit(void) {
> + int i = 0;
> + char *tmp;
> + int rc = 0;
> +
> + for(i = 0; i <RDT_NUM_RESOURCES; i ++) {
> + if ((rc = asprintf(&tmp, "%s/%s", RESCTRL_INFO_DIR, ResCtrlAll[i].name)) < 0) {
> + continue;
> + }
> + if (virFileExists(tmp)) {
> + if ((rc = virResCtrlGetConfig(i)) < 0 )
> + VIR_WARN("Ignor error while get config for %d", i);
Typo.
> + }
> +
> + VIR_FREE(tmp);
> + }
> + return rc;
> +}
> +
> +bool virResCtrlAvailable(void) {
> + if (!virFileExists(RESCTRL_INFO_DIR))
> + return false;
> + return true;
> +}
> +
> +virResCtrlPtr virResCtrlGet(int type) {
> + return &ResCtrlAll[type];
> +}
> diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
> new file mode 100644
> index 0000000..432f191
> --- /dev/null
> +++ b/src/util/virresctrl.h
> @@ -0,0 +1,121 @@
> +/*
> + * * virrscctrl.h: methods for managing rscctrl
> + * *
> + * * Copyright (C) 2016 Intel, Inc.
> + * *
> + * * This library is free software; you can redistribute it and/or
> + * * modify it under the terms of the GNU Lesser General Public
> + * * License as published by the Free Software Foundation; either
> + * * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library. If not, see
> + * <http://www.gnu.org/licenses/>.
> + *
> + * Authors:
> + * Eli Qiao <liyong.qiao@xxxxxxxxx>
> + */
> +
> +#ifndef __VIR_RESCTRL_H__
> +# define __VIR_RESCTRL_H__
> +
> +# include "virutil.h"
> +# include "virbitmap.h"
> +# include "domain_conf.h"
> +
> +#define RESCTRL_DIR "/sys/fs/resctrl"
> +#define RESCTRL_INFO_DIR "/sys/fs/resctrl/info"
> +#define SYSFS_SYSTEM_PATH "/sys/devices/system"
> +
> +#define MAX_CPU_SOCKET_NUM 8
> +#define MAX_CBM_BIT_LEN 32
> +#define MAX_SCHEMATA_LEN 1024
> +#define MAX_FILE_LEN ( 10 * 1024 * 1024)
> +
> +enum {
> + RDT_RESOURCE_L3,
> + RDT_RESOURCE_L3DATA,
> + RDT_RESOURCE_L3CODE,
> + RDT_RESOURCE_L2,
> + /* Must be the last */
> + RDT_NUM_RESOURCES,
> +};
> +
> +/**
> + * a virResSchemata represents a schemata object under a resource control
> + * domain.
> + */
> +typedef struct _virResSchemata virResSchemata;
> +typedef virResSchemata *virResSchemataPtr;
> +
> +struct _virResSchemata {
> + char *name;
> + unsigned int socket_no;
> + int schemata;
> +};
> +
> +/**
> + * a virResDomain represents a resource control domain. It's a double linked
> + * list.
> + */
> +
> +typedef struct _virResDomain virResDomain;
> +typedef virResDomain *virResDomainPtr;
> +
> +struct _virResDomain {
> + char* name;
> + virResSchemataPtr schematas;
> + char* tasks;
> + int n_sockets;
> + virResDomainPtr pre;
> + virResDomainPtr next;
> +};
> +
> +/**
> + * struct rdt_resource - attributes of an RDT resource
> + * @enabled: Is this feature enabled on this machine
> + * @name: Name to use in "schemata" file
> + * @num_closid: Number of CLOSIDs available
> + * @max_cbm: Largest Cache Bit Mask allowed
> + * @min_cbm_bits: Minimum number of consecutive bits to be set
> + * in a cache bit mask
> + * @domains: All resource domains for this resource
> + * @num_domains: Number of domains active
> + * @num_tmp_cbms: Number of CBMs in tmp_cbms
> + * @cache_level: Which cache level defines scope of this domain
> + * @num_sockets: Number of sockets on this machine.
> + * @cache_size: Cache size on each cpu socket in KiB.
> + * @cache_left: Cache left on each cpu socket in KiB.
> + * @cache_min: The minimum cache of allocatioin.
> + * @cpu_mask: cpu mask on each socket.
> + */
> +typedef struct _virResCtrl virResCtrl;
> +typedef virResCtrl *virResCtrlPtr;
> +
> +struct _virResCtrl {
> + bool enabled;
> + const char *name;
> + int num_closid;
> + int cbm_len;
> + int min_cbm_bits;
> + virResDomainPtr domains;
> + int num_domains;
> + const char* cache_level;
> + int num_sockets;
> + int cache_size[MAX_CPU_SOCKET_NUM];
> + int cache_left[MAX_CPU_SOCKET_NUM];
> + int cache_min[MAX_CPU_SOCKET_NUM];
> + virBitmapPtr cpu_mask[MAX_CPU_SOCKET_NUM];
> +};
> +
> +
> +bool virResCtrlAvailable(void);
> +int virResCtrlInit(void);
> +virResCtrlPtr virResCtrlGet(int);
> +
> +#endif
> diff --git a/tests/Makefile.am b/tests/Makefile.am
> index ecd04e8..e9c9abc 100644
> --- a/tests/Makefile.am
> +++ b/tests/Makefile.am
> @@ -1185,6 +1185,10 @@ virrotatingfiletest_SOURCES = \
> virrotatingfiletest_CFLAGS = $(AM_CFLAGS)
> virrotatingfiletest_LDADD = $(LDADDS)
>
> +virresctrltest_SOURCES = \
> + virresctrltest.c testutils.h testutils.c
> +virresctrltest_LDADD = $(LDADDS)
> +
> if WITH_LINUX
> virusbtest_SOURCES = \
> virusbtest.c testutils.h testutils.c
> --
> 1.9.1
--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list
