[PATCH] Add NUMA memory and CPU thread siblings to capabilities

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



---
 docs/schemas/capability.rng               |  15 +++
 src/conf/capabilities.c                   |  65 ++++++++++---
 src/conf/capabilities.h                   |   7 +-
 src/nodeinfo.c                            | 155 +++++++++++++++++++++++++++++-
 src/test/test_driver.c                    |   2 +-
 src/xen/xend_internal.c                   |   4 +-
 tests/capabilityschemadata/caps-test3.xml |  88 +++++++++++++++++
 7 files changed, 319 insertions(+), 17 deletions(-)
 create mode 100644 tests/capabilityschemadata/caps-test3.xml

diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng
index 8c928bc..1ab36ed 100644
--- a/docs/schemas/capability.rng
+++ b/docs/schemas/capability.rng
@@ -176,6 +176,10 @@
       </attribute>
 
       <optional>
+        <ref name='memory'/>
+      </optional>
+
+      <optional>
         <element name='cpus'>
           <attribute name='num'>
             <ref name='unsignedInt'/>
@@ -188,11 +192,22 @@
     </element>
   </define>
 
+  <define name='memory'>
+    <element name='memory'>
+        <ref name='scaledInteger'/>
+    </element>
+  </define>
+
   <define name='cpu'>
     <element name='cpu'>
       <attribute name='id'>
         <ref name='unsignedInt'/>
       </attribute>
+      <optional>
+        <attribute name='thread_siblings'>
+          <text/>
+        </attribute>
+      </optional>
     </element>
   </define>
 
diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c
index a8ee2cf..1b514ef 100644
--- a/src/conf/capabilities.c
+++ b/src/conf/capabilities.c
@@ -73,10 +73,16 @@ virCapabilitiesNew(const char *arch,
 static void
 virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell)
 {
+    int i;
     if (cell == NULL)
         return;
 
+    if (cell->threadSiblings)
+        for (i=0; i < cell->ncpus; i++)
+            VIR_FREE(cell->threadSiblings[i]);
+
     VIR_FREE(cell->cpus);
+    VIR_FREE(cell->threadSiblings);
     VIR_FREE(cell);
 }
 
@@ -253,7 +259,10 @@ int
 virCapabilitiesAddHostNUMACell(virCapsPtr caps,
                                int num,
                                int ncpus,
-                               const int *cpus)
+                               unsigned long long mem,
+                               const int *cpus,
+                               const virBitmapPtr *threadSiblings
+)
 {
     virCapsHostNUMACellPtr cell;
 
@@ -272,8 +281,20 @@ virCapabilitiesAddHostNUMACell(virCapsPtr caps,
            cpus,
            ncpus * sizeof(*cpus));
 
+    if (threadSiblings) {
+        if (VIR_ALLOC_N(cell->threadSiblings, ncpus) < 0) {
+            VIR_FREE(cell->cpus);
+            VIR_FREE(cell);
+            return -1;
+        }
+        memcpy(cell->threadSiblings,
+               threadSiblings,
+               ncpus * sizeof(*threadSiblings));
+    }
+
     cell->ncpus = ncpus;
     cell->num = num;
+    cell->mem = mem;
 
     caps->host.numaCell[caps->host.nnumaCell++] = cell;
 
@@ -695,6 +716,7 @@ virCapabilitiesFormatXML(virCapsPtr caps)
     virBuffer xml = VIR_BUFFER_INITIALIZER;
     int i, j, k;
     char host_uuid[VIR_UUID_STRING_BUFLEN];
+    char *str;
 
     virBufferAddLit(&xml, "<capabilities>\n\n");
     virBufferAddLit(&xml, "  <host>\n");
@@ -754,22 +776,43 @@ virCapabilitiesFormatXML(virCapsPtr caps)
     }
 
     if (caps->host.nnumaCell) {
-        virBufferAddLit(&xml, "    <topology>\n");
-        virBufferAsprintf(&xml, "      <cells num='%zu'>\n",
+        virBufferAdjustIndent(&xml, 4);
+        virBufferAddLit(&xml, "<topology>\n");
+        virBufferAsprintf(&xml, "  <cells num='%zu'>\n",
                           caps->host.nnumaCell);
         for (i = 0 ; i < caps->host.nnumaCell ; i++) {
-            virBufferAsprintf(&xml, "        <cell id='%d'>\n",
+            virBufferAsprintf(&xml, "    <cell id='%d'>\n",
                               caps->host.numaCell[i]->num);
-            virBufferAsprintf(&xml, "          <cpus num='%d'>\n",
+
+            /* Print out the numacell memory total if it is available */
+            if (caps->host.numaCell[i]->mem)
+                virBufferAsprintf(&xml, "      <memory unit='KiB'>%llu</memory>\n",
+                                  caps->host.numaCell[i]->mem);
+
+            virBufferAsprintf(&xml, "      <cpus num='%d'>\n",
                               caps->host.numaCell[i]->ncpus);
-            for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++)
-                virBufferAsprintf(&xml, "            <cpu id='%d'/>\n",
+            for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++) {
+                virBufferAsprintf(&xml, "        <cpu id='%d'",
                                   caps->host.numaCell[i]->cpus[j]);
-            virBufferAddLit(&xml, "          </cpus>\n");
-            virBufferAddLit(&xml, "        </cell>\n");
+
+                /* Print out thread siblings if they were populated */
+                if (caps->host.numaCell[i]->threadSiblings) {
+                    str = virBitmapFormat(caps->host.numaCell[i]->threadSiblings[j]);
+                    if (str) {
+                        virBufferAsprintf(&xml, " thread_siblings='%s'", str);
+                        VIR_FREE(str);
+                    }
+                }
+
+                virBufferAsprintf(&xml, "/>\n");
+            }
+
+            virBufferAddLit(&xml, "      </cpus>\n");
+            virBufferAddLit(&xml, "    </cell>\n");
         }
-        virBufferAddLit(&xml, "      </cells>\n");
-        virBufferAddLit(&xml, "    </topology>\n");
+        virBufferAddLit(&xml, "  </cells>\n");
+        virBufferAddLit(&xml, "</topology>\n");
+        virBufferAdjustIndent(&xml, -4);
     }
 
     for (i = 0; i < caps->host.nsecModels; i++) {
diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h
index 99056f8..ded0d57 100644
--- a/src/conf/capabilities.h
+++ b/src/conf/capabilities.h
@@ -88,7 +88,9 @@ typedef virCapsHostNUMACell *virCapsHostNUMACellPtr;
 struct _virCapsHostNUMACell {
     int num;
     int ncpus;
+    unsigned long long mem; /* in kibibytes */
     int *cpus;
+    virBitmapPtr *threadSiblings;
 };
 
 typedef struct _virCapsHostSecModel virCapsHostSecModel;
@@ -200,7 +202,10 @@ extern int
 virCapabilitiesAddHostNUMACell(virCapsPtr caps,
                                int num,
                                int ncpus,
-                               const int *cpus);
+                               unsigned long long mem,
+                               const int *cpus,
+                               const virBitmapPtr *threadSiblings
+);
 
 
 extern int
diff --git a/src/nodeinfo.c b/src/nodeinfo.c
index 36fbd66..cc42c43 100644
--- a/src/nodeinfo.c
+++ b/src/nodeinfo.c
@@ -56,6 +56,7 @@
 #ifdef __linux__
 # define CPUINFO_PATH "/proc/cpuinfo"
 # define SYSFS_SYSTEM_PATH "/sys/devices/system"
+# define SYSFS_CPU_PATH "/sys/devices/system/cpu"
 # define PROCSTAT_PATH "/proc/stat"
 # define MEMINFO_PATH "/proc/meminfo"
 # define SYSFS_MEMORY_SHARED_PATH "/sys/kernel/mm/ksm"
@@ -77,6 +78,7 @@ static int linuxNodeGetMemoryStats(FILE *meminfo,
                                    int cellNum,
                                    virNodeMemoryStatsPtr params,
                                    int *nparams);
+static unsigned long long nodeGetCellMemory(int cell);
 
 /* Return the positive decimal contents of the given
  * DIR/cpu%u/FILE, or -1 on error.  If MISSING_OK and the
@@ -125,6 +127,66 @@ cleanup:
     return value;
 }
 
+/**
+ * virNodeGetThreadSiblingsList
+ * @dir: directory where cpu0-N files are located.
+ * @cpu: the specific cpu to get the siblings for.
+ *
+ * Will open the "thread_siblings_list" file for the cpu and
+ * return a string representing the contents. The contents of the
+ * file is a string represnting the cpus that are siblings; like
+ * 1,9 or 1-4.
+ *
+ * Returns NULL on failure, char * on success
+ *
+ * Note: Responsibility of caller to free string
+ */
+static char *virNodeGetThreadSiblingsList(const char *dir, unsigned int cpu)
+{
+    char *path;
+    FILE *pathfp;
+    char *str = NULL;
+    int strsize;
+
+    if (virAsprintf(&path, "%s/cpu%u/topology/thread_siblings_list",
+                    dir, cpu) < 0) {
+        virReportOOMError();
+        return str;
+    }
+
+    pathfp = fopen(path, "r");
+    if (pathfp == NULL) {
+        virReportSystemError(errno, _("cannot open %s"), path);
+        VIR_FREE(path);
+        return str;
+    }
+
+    /* Detect how large of a string we need to make */
+    fseek(pathfp, 0L, SEEK_END);
+    strsize = ftell(pathfp);
+    fseek(pathfp, 0L, SEEK_SET);
+
+
+    if (VIR_ALLOC_N(str, strsize) < 0) {
+        virReportOOMError();
+        str = NULL;
+        goto cleanup;
+    }
+
+    if (fgets(str, strsize, pathfp) == NULL) {
+        virReportSystemError(errno, _("cannot read from %s"), path);
+        VIR_FREE(str);
+        str = NULL;
+        goto cleanup;
+    }
+
+cleanup:
+    VIR_FORCE_FCLOSE(pathfp);
+    VIR_FREE(path);
+
+    return str;
+}
+
 static unsigned long
 virNodeCountThreadSiblings(const char *dir, unsigned int cpu)
 {
@@ -1313,9 +1375,14 @@ nodeCapsInitNUMA(virCapsPtr caps)
     int n;
     unsigned long *mask = NULL;
     unsigned long *allonesmask = NULL;
+    unsigned long long memory;
     int *cpus = NULL;
     int ret = -1;
     int max_n_cpus = NUMA_MAX_N_CPUS;
+    char *str = NULL;
+    virBitmapPtr *threadSiblings = NULL;
+
+
 
     if (numa_available() < 0)
         return 0;
@@ -1343,29 +1410,56 @@ nodeCapsInitNUMA(virCapsPtr caps)
             continue;
         }
 
+        /* Detect the amount of memory in the numa cell */
+        memory = nodeGetCellMemory(n);
+        if (memory == 0)
+            goto cleanup;
+
         for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++)
             if (MASK_CPU_ISSET(mask, i))
                 ncpus++;
 
+        /* Create some memory for the array of cpus. */
         if (VIR_ALLOC_N(cpus, ncpus) < 0)
             goto cleanup;
 
-        for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++)
-            if (MASK_CPU_ISSET(mask, i))
+        /* Create some memory for the array of siblings. */
+        if (VIR_ALLOC_N(threadSiblings, ncpus) < 0)
+            goto cleanup;
+
+        for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) {
+            if (MASK_CPU_ISSET(mask, i)) {
+
+                /* Get the string of thread siblings for this cpu */
+                if (!(str = virNodeGetThreadSiblingsList(SYSFS_CPU_PATH, i)))
+                    goto cleanup;
+
+                /* Convert the string to a bitmap to be stored */
+                if (virBitmapParse(str, 0, &threadSiblings[ncpus], max_n_cpus) < 0)
+                    goto cleanup;
+
                 cpus[ncpus++] = i;
+                VIR_FREE(str);
+            }
+        }
 
         if (virCapabilitiesAddHostNUMACell(caps,
                                            n,
                                            ncpus,
-                                           cpus) < 0)
+                                           memory,
+                                           cpus,
+                                           threadSiblings) < 0)
             goto cleanup;
 
+        VIR_FREE(threadSiblings);
         VIR_FREE(cpus);
     }
 
     ret = 0;
 
 cleanup:
+    VIR_FREE(str);
+    VIR_FREE(threadSiblings);
     VIR_FREE(cpus);
     VIR_FREE(mask);
     VIR_FREE(allonesmask);
@@ -1441,6 +1535,54 @@ cleanup:
     return freeMem;
 }
 
+/**
+ * nodeGetCellMemory
+ * @cell: The number of the numa cell to get memory info for.
+ *
+ * Will call the numa_node_size64() function from libnuma to get
+ * the amount of total memory in bytes. It is then converted to
+ * KiB and returned.
+ *
+ * Returns 0 on failure, amount of memory in KiB on success.
+ */
+static unsigned long long
+nodeGetCellMemory(int cell)
+{
+    long long mem;
+    unsigned long long memKiB = 0;
+    int maxCell;
+
+    if (numa_available() < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       "%s", _("NUMA not supported on this host"));
+        goto cleanup;
+    }
+
+    /* Make sure the provided cell number is valid. */
+    maxCell = numa_max_node();
+    if (cell > maxCell) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("cell %d out of range (0-%d)"),
+                       cell, maxCell);
+        goto cleanup;
+    }
+
+    /* Get the amount of memory(bytes) in the node */
+    mem = numa_node_size64(cell, NULL);
+    if (mem < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("Failed to query NUMA total memory for node: %d"),
+                       cell);
+        goto cleanup;
+    }
+
+    /* Convert the memory from bytes to KiB */
+    memKiB = mem >> 10;
+
+cleanup:
+    return memKiB;
+}
+
 #else
 int nodeCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) {
     return 0;
@@ -1462,4 +1604,11 @@ unsigned long long nodeGetFreeMemory(virConnectPtr conn ATTRIBUTE_UNUSED)
                    _("NUMA memory information not available on this platform"));
     return 0;
 }
+
+static unsigned long long nodeGetCellMemory(int cell)
+{
+    virReportError(VIR_ERR_NO_SUPPORT, "%s",
+                   _("NUMA memory information not available on this platform"));
+    return 0;
+}
 #endif
diff --git a/src/test/test_driver.c b/src/test/test_driver.c
index c974e1a..9d715a0 100644
--- a/src/test/test_driver.c
+++ b/src/test/test_driver.c
@@ -174,7 +174,7 @@ testBuildCapabilities(virConnectPtr conn) {
 
     for (i = 0; i < privconn->numCells; i++) {
         if (virCapabilitiesAddHostNUMACell(caps, i, privconn->cells[i].numCpus,
-                                           privconn->cells[i].cpus) < 0)
+                                           0, privconn->cells[i].cpus, NULL) < 0)
             goto no_memory;
     }
 
diff --git a/src/xen/xend_internal.c b/src/xen/xend_internal.c
index 922c571..d18eecb 100644
--- a/src/xen/xend_internal.c
+++ b/src/xen/xend_internal.c
@@ -1167,7 +1167,9 @@ sexpr_to_xend_topology(const struct sexpr *root,
         if (virCapabilitiesAddHostNUMACell(caps,
                                            cell,
                                            nb_cpus,
-                                           cpuNums) < 0)
+                                           0,
+                                           cpuNums,
+                                           NULL) < 0)
             goto memory_error;
     }
     VIR_FREE(cpuNums);
diff --git a/tests/capabilityschemadata/caps-test3.xml b/tests/capabilityschemadata/caps-test3.xml
new file mode 100644
index 0000000..724ced4
--- /dev/null
+++ b/tests/capabilityschemadata/caps-test3.xml
@@ -0,0 +1,88 @@
+<capabilities>
+
+  <host>
+    <uuid>35383339-3134-5553-4531-30314e394a50</uuid>
+    <cpu>
+      <arch>x86_64</arch>
+      <model>Westmere</model>
+      <vendor>Intel</vendor>
+      <topology sockets='1' cores='6' threads='2'/>
+      <feature name='rdtscp'/>
+      <feature name='pdpe1gb'/>
+      <feature name='dca'/>
+      <feature name='pdcm'/>
+      <feature name='xtpr'/>
+      <feature name='tm2'/>
+      <feature name='est'/>
+      <feature name='smx'/>
+      <feature name='vmx'/>
+      <feature name='ds_cpl'/>
+      <feature name='monitor'/>
+      <feature name='dtes64'/>
+      <feature name='pclmuldq'/>
+      <feature name='pbe'/>
+      <feature name='tm'/>
+      <feature name='ht'/>
+      <feature name='ss'/>
+      <feature name='acpi'/>
+      <feature name='ds'/>
+      <feature name='vme'/>
+    </cpu>
+    <power_management>
+      <suspend_disk/>
+    </power_management>
+    <migration_features>
+      <live/>
+      <uri_transports>
+        <uri_transport>tcp</uri_transport>
+      </uri_transports>
+    </migration_features>
+    <topology>
+      <cells num='2'>
+        <cell id='0'>
+          <memory unit='KiB'>12572412</memory>
+          <cpus num='12'>
+            <cpu id='0' thread_siblings='0,12'/>
+            <cpu id='2' thread_siblings='2,14'/>
+            <cpu id='4' thread_siblings='4,16'/>
+            <cpu id='6' thread_siblings='6,18'/>
+            <cpu id='8' thread_siblings='8,20'/>
+            <cpu id='10' thread_siblings='10,22'/>
+            <cpu id='12' thread_siblings='0,12'/>
+            <cpu id='14' thread_siblings='2,14'/>
+            <cpu id='16' thread_siblings='4,16'/>
+            <cpu id='18' thread_siblings='6,18'/>
+            <cpu id='20' thread_siblings='8,20'/>
+            <cpu id='22' thread_siblings='10,22'/>
+          </cpus>
+        </cell>
+        <cell id='1'>
+          <memory unit='KiB'>12582908</memory>
+          <cpus num='12'>
+            <cpu id='1' thread_siblings='1,13'/>
+            <cpu id='3' thread_siblings='3,15'/>
+            <cpu id='5' thread_siblings='5,17'/>
+            <cpu id='7' thread_siblings='7,19'/>
+            <cpu id='9' thread_siblings='9,21'/>
+            <cpu id='11' thread_siblings='11,23'/>
+            <cpu id='13' thread_siblings='1,13'/>
+            <cpu id='15' thread_siblings='3,15'/>
+            <cpu id='17' thread_siblings='5,17'/>
+            <cpu id='19' thread_siblings='7,19'/>
+            <cpu id='21' thread_siblings='9,21'/>
+            <cpu id='23' thread_siblings='11,23'/>
+          </cpus>
+        </cell>
+      </cells>
+    </topology>
+    <secmodel>
+      <model>none</model>
+      <doi>0</doi>
+    </secmodel>
+    <secmodel>
+      <model>dac</model>
+      <doi>0</doi>
+    </secmodel>
+  </host>
+
+</capabilities>
-- 
1.7.11.7

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list


[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]