[PATCH 3/3] virCaps: Expose distance between host NUMA nodes

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



If user or management application wants to create a guest,
it may be useful to know the cost of internode latencies
before the guest resources are pinned. For example:

<capabilities>

  <host>
    ...
    <topology>
      <cells num='2'>
        <cell id='0'>
          <memory unit='KiB'>4004132</memory>
          <cell id='0' distance='10'/>
          <cell id='1' distance='20'/>
          <cpus num='2'>
            <cpu id='0' socket_id='0' core_id='0' siblings='0'/>
            <cpu id='2' socket_id='0' core_id='2' siblings='2'/>
          </cpus>
        </cell>
        <cell id='1'>
          <memory unit='KiB'>4030064</memory>
          <cell id='0' distance='20'/>
          <cell id='1' distance='10'/>
          <cpus num='2'>
            <cpu id='1' socket_id='0' core_id='0' siblings='1'/>
            <cpu id='3' socket_id='0' core_id='2' siblings='3'/>
          </cpus>
        </cell>
      </cells>
    </topology>
    ...
  </host>
  ...
</capabilities>

we can see the distance from node1 to node0 is 20 and within nodes 10.

Signed-off-by: Michal Privoznik <mprivozn@xxxxxxxxxx>
---
 docs/schemas/capability.rng | 11 ++++++++
 src/conf/capabilities.c     | 13 +++++++++-
 src/conf/capabilities.h     | 13 +++++++++-
 src/libxl/libxl_conf.c      |  4 +--
 src/nodeinfo.c              | 61 ++++++++++++++++++++++++++++++++++++++++++---
 src/test/test_driver.c      |  2 +-
 src/xen/xend_internal.c     |  3 ++-
 tests/vircapstest.c         |  4 +--
 8 files changed, 100 insertions(+), 11 deletions(-)

diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng
index d2d9776..63b53b9 100644
--- a/docs/schemas/capability.rng
+++ b/docs/schemas/capability.rng
@@ -188,6 +188,17 @@
         <ref name='memory'/>
       </optional>
 
+      <zeroOrMore>
+        <element name='cell'>
+          <attribute name='id'>
+            <ref name='unsignedInt'/>
+          </attribute>
+          <attribute name='distance'>
+            <ref name='unsignedInt'/>
+          </attribute>
+        </element>
+      </zeroOrMore>
+
       <optional>
         <element name='cpus'>
           <attribute name='num'>
diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c
index cf474d7..e66abb2 100644
--- a/src/conf/capabilities.c
+++ b/src/conf/capabilities.c
@@ -107,6 +107,7 @@ virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell)
     virCapabilitiesClearHostNUMACellCPUTopology(cell->cpus, cell->ncpus);
 
     VIR_FREE(cell->cpus);
+    VIR_FREE(cell->siblings);
     VIR_FREE(cell);
 }
 
@@ -286,8 +287,10 @@ int
 virCapabilitiesAddHostNUMACell(virCapsPtr caps,
                                int num,
                                int ncpus,
+                               int nsiblings,
                                unsigned long long mem,
-                               virCapsHostNUMACellCPUPtr cpus)
+                               virCapsHostNUMACellCPUPtr cpus,
+                               virCapsHostNUMACellSiblingInfoPtr siblings)
 {
     virCapsHostNUMACellPtr cell;
 
@@ -302,6 +305,8 @@ virCapabilitiesAddHostNUMACell(virCapsPtr caps,
     cell->num = num;
     cell->mem = mem;
     cell->cpus = cpus;
+    cell->siblings = siblings;
+    cell->nsiblings = nsiblings;
 
     caps->host.numaCell[caps->host.nnumaCell++] = cell;
 
@@ -766,6 +771,12 @@ virCapabilitiesFormatNUMATopology(virBufferPtr buf,
             virBufferAsprintf(buf, "<memory unit='KiB'>%llu</memory>\n",
                               cells[i]->mem);
 
+        for (j = 0; j < cells[i]->nsiblings; j++) {
+            virBufferAsprintf(buf, "<cell id='%d' distance='%d'/>\n",
+                              cells[i]->siblings[j].node,
+                              cells[i]->siblings[j].distance);
+        }
+
         virBufferAsprintf(buf, "<cpus num='%d'>\n", cells[i]->ncpus);
         virBufferAdjustIndent(buf, 2);
         for (j = 0; j < cells[i]->ncpus; j++) {
diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h
index ba99e1a..5da31a8 100644
--- a/src/conf/capabilities.h
+++ b/src/conf/capabilities.h
@@ -95,6 +95,13 @@ struct _virCapsHostNUMACellCPU {
     virBitmapPtr siblings;
 };
 
+typedef struct _virCapsHostNUMACellSiblingInfo virCapsHostNUMACellSiblingInfo;
+typedef virCapsHostNUMACellSiblingInfo *virCapsHostNUMACellSiblingInfoPtr;
+struct _virCapsHostNUMACellSiblingInfo {
+    int node;               /* foreign NUMA node */
+    unsigned int distance;  /* distance to the node */
+};
+
 typedef struct _virCapsHostNUMACell virCapsHostNUMACell;
 typedef virCapsHostNUMACell *virCapsHostNUMACellPtr;
 struct _virCapsHostNUMACell {
@@ -102,6 +109,8 @@ struct _virCapsHostNUMACell {
     int ncpus;
     unsigned long long mem; /* in kibibytes */
     virCapsHostNUMACellCPUPtr cpus;
+    int nsiblings;
+    virCapsHostNUMACellSiblingInfoPtr siblings;
 };
 
 typedef struct _virCapsHostSecModelLabel virCapsHostSecModelLabel;
@@ -194,8 +203,10 @@ extern int
 virCapabilitiesAddHostNUMACell(virCapsPtr caps,
                                int num,
                                int ncpus,
+                               int nsiblings,
                                unsigned long long mem,
-                               virCapsHostNUMACellCPUPtr cpus);
+                               virCapsHostNUMACellCPUPtr cpus,
+                               virCapsHostNUMACellSiblingInfoPtr siblings);
 
 
 extern int
diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
index b7fed7f..1db570d 100644
--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
@@ -208,8 +208,8 @@ libxlCapsInitNuma(libxl_ctx *ctx, virCapsPtr caps)
             continue;
 
         if (virCapabilitiesAddHostNUMACell(caps, i, nr_cpus_node[i],
-                                           numa_info[i].size / 1024,
-                                           cpus[i]) < 0) {
+                                           0, numa_info[i].size / 1024,
+                                           cpus[i], NULL) < 0) {
             virCapabilitiesClearHostNUMACellCPUTopology(cpus[i],
                                                         nr_cpus_node[i]);
             goto cleanup;
diff --git a/src/nodeinfo.c b/src/nodeinfo.c
index 56f2b02..f76a91e 100644
--- a/src/nodeinfo.c
+++ b/src/nodeinfo.c
@@ -1644,9 +1644,9 @@ nodeCapsInitNUMAFake(virCapsPtr caps ATTRIBUTE_UNUSED)
     }
 
     if (virCapabilitiesAddHostNUMACell(caps, 0,
-                                       ncpus,
+                                       ncpus, 0,
                                        nodeinfo.memory,
-                                       cpus) < 0)
+                                       cpus, NULL) < 0)
         goto error;
 
     return 0;
@@ -1748,6 +1748,53 @@ virNodeCapsFillCPUInfo(int cpu_id ATTRIBUTE_UNUSED,
 #endif
 }
 
+static int
+virNodeCapsGetSiblingInfo(int node,
+                          virCapsHostNUMACellSiblingInfoPtr *siblings,
+                          int *nsiblings)
+{
+    virCapsHostNUMACellSiblingInfoPtr tmp = NULL;
+    int tmp_size = 0;
+    int ret = -1;
+    int *distances = NULL;
+    int ndistances = 0;
+    size_t i;
+
+    if (virNumaGetDistances(node, &distances, &ndistances) < 0)
+        goto cleanup;
+
+    if (!distances) {
+        *siblings = NULL;
+        *nsiblings = 0;
+        return 0;
+    }
+
+    if (VIR_ALLOC_N(tmp, ndistances) < 0)
+        goto cleanup;
+
+    for (i = 0; i < ndistances; i++) {
+        if (!distances[i])
+            continue;
+
+        tmp[tmp_size].node = i;
+        tmp[tmp_size].distance = distances[i];
+        tmp_size++;
+    }
+
+    if (VIR_REALLOC_N(tmp, tmp_size) < 0)
+        goto cleanup;
+
+    *siblings = tmp;
+    *nsiblings = tmp_size;
+    tmp = NULL;
+    tmp_size = 0;
+    ret = 0;
+ cleanup:
+    VIR_FREE(distances);
+    VIR_FREE(tmp);
+    return ret;
+}
+
 int
 nodeCapsInitNUMA(virCapsPtr caps)
 {
@@ -1755,6 +1802,8 @@ nodeCapsInitNUMA(virCapsPtr caps)
     unsigned long long memory;
     virCapsHostNUMACellCPUPtr cpus = NULL;
     virBitmapPtr cpumap = NULL;
+    virCapsHostNUMACellSiblingInfoPtr siblings = NULL;
+    int nsiblings;
     int ret = -1;
     int ncpus = 0;
     int cpu;
@@ -1794,14 +1843,19 @@ nodeCapsInitNUMA(virCapsPtr caps)
             }
         }
 
+        if (virNodeCapsGetSiblingInfo(n, &siblings, &nsiblings) < 0)
+            goto cleanup;
+
         /* Detect the amount of memory in the numa cell in KiB */
         virNumaGetNodeMemory(n, &memory, NULL);
         memory >>= 10;
 
-        if (virCapabilitiesAddHostNUMACell(caps, n, ncpus, memory, cpus) < 0)
+        if (virCapabilitiesAddHostNUMACell(caps, n, ncpus, nsiblings,
+                                           memory, cpus, siblings) < 0)
             goto cleanup;
 
         cpus = NULL;
+        siblings = NULL;
     }
 
     ret = 0;
@@ -1812,6 +1866,7 @@ nodeCapsInitNUMA(virCapsPtr caps)
 
     virBitmapFree(cpumap);
     VIR_FREE(cpus);
+    VIR_FREE(siblings);
 
     if (ret < 0)
         VIR_FREE(cpus);
diff --git a/src/test/test_driver.c b/src/test/test_driver.c
index 37756e7..b204e7d 100644
--- a/src/test/test_driver.c
+++ b/src/test/test_driver.c
@@ -337,7 +337,7 @@ testBuildCapabilities(virConnectPtr conn)
 
 
         if (virCapabilitiesAddHostNUMACell(caps, i, privconn->cells[i].numCpus,
-                                           0, cpu_cells) < 0)
+                                           0, 0, cpu_cells, NULL) < 0)
             goto error;
     }
 
diff --git a/src/xen/xend_internal.c b/src/xen/xend_internal.c
index c30b136..f4c824e 100644
--- a/src/xen/xend_internal.c
+++ b/src/xen/xend_internal.c
@@ -1100,7 +1100,8 @@ sexpr_to_xend_topology(const struct sexpr *root, virCapsPtr caps)
         }
         virBitmapFree(cpuset);
 
-        if (virCapabilitiesAddHostNUMACell(caps, cell, nb_cpus, 0, cpuInfo) < 0)
+        if (virCapabilitiesAddHostNUMACell(caps, cell, nb_cpus,
+                                           0, 0, cpuInfo, NULL) < 0)
             goto error;
         cpuInfo = NULL;
     }
diff --git a/tests/vircapstest.c b/tests/vircapstest.c
index 821a92b..9df074a 100644
--- a/tests/vircapstest.c
+++ b/tests/vircapstest.c
@@ -64,9 +64,9 @@ buildNUMATopology(int seq)
         id++;
 
         if (virCapabilitiesAddHostNUMACell(caps, cell_id + seq,
-                                           MAX_CPUS_IN_CELL,
+                                           MAX_CPUS_IN_CELL, 0,
                                            MAX_MEM_IN_CELL,
-                                           cell_cpus) < 0)
+                                           cell_cpus, NULL) < 0)
            goto error;
 
         cell_cpus = NULL;
-- 
2.0.0

--
libvir-list mailing list
libvir-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/libvir-list




[Index of Archives]     [Virt Tools]     [Libvirt Users]     [Lib OS Info]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Big List of Linux Books]     [Yosemite News]     [KDE Users]     [Fedora Tools]