When running applications on the machine with NVDIMM as NUMA node, the
memory allocation may end up on NVDIMM node. This may result in silent
performance degradation and regression due to the difference of hardware
property.
DRAM first should be obeyed to prevent from surprising regression. Any
non-DRAM nodes should be excluded from default allocation. Use nodemask
to control the memory placement. Introduce def_alloc_nodemask which has
DRAM nodes set only. Any non-DRAM allocation should be specified by
NUMA policy explicitly.
In the future we may be able to extract the memory charasteristics from
HMAT or other source to build up the default allocation nodemask.
However, just distinguish DRAM and PMEM (non-DRAM) nodes by SRAT flag
for the time being.
Signed-off-by: Yang Shi <yang.shi@xxxxxxxxxxxxxxxxx>
---
arch/x86/mm/numa.c | 1 +
drivers/acpi/numa.c | 8 ++++++++
include/linux/mmzone.h | 3 +++
mm/page_alloc.c | 18 ++++++++++++++++--
4 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index dfb6c4d..d9e0ca4 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -626,6 +626,7 @@ static int __init numa_init(int (*init_func)(void))
nodes_clear(numa_nodes_parsed);
nodes_clear(node_possible_map);
nodes_clear(node_online_map);
+ nodes_clear(def_alloc_nodemask);
memset(&numa_meminfo, 0, sizeof(numa_meminfo));
WARN_ON(memblock_set_node(0, ULLONG_MAX, &memblock.memory,
MAX_NUMNODES));
diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c
index 867f6e3..79dfedf 100644
--- a/drivers/acpi/numa.c
+++ b/drivers/acpi/numa.c
@@ -296,6 +296,14 @@ void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
goto out_err_bad_srat;
}
+ /*
+ * Non volatile memory is excluded from zonelist by default.
+ * Only regular DRAM nodes are set in default allocation node
+ * mask.
+ */
+ if (!(ma->flags & ACPI_SRAT_MEM_NON_VOLATILE))
+ node_set(node, def_alloc_nodemask);
+
node_set(node, numa_nodes_parsed);
pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index fba7741..063c3b4 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -927,6 +927,9 @@ extern int numa_zonelist_order_handler(struct ctl_table *, int,
extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
extern struct zone *next_zone(struct zone *zone);
+/* Regular DRAM nodes */
+extern nodemask_t def_alloc_nodemask;
+
/**
* for_each_online_pgdat - helper macro to iterate over all online nodes
* @pgdat - pointer to a pg_data_t variable
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 03fcf73..68ad8c6 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -134,6 +134,8 @@ struct pcpu_drain {
int percpu_pagelist_fraction;
gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
+nodemask_t def_alloc_nodemask __read_mostly;
+
/*
* A cached value of the page's pageblock's migratetype, used when the page is
* put on a pcplist. Used to avoid the pageblock migratetype lookup when
@@ -4524,12 +4526,24 @@ static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
{
ac->high_zoneidx = gfp_zone(gfp_mask);
ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
- ac->nodemask = nodemask;
ac->migratetype = gfpflags_to_migratetype(gfp_mask);
+ if (!nodemask) {
+ /* Non-DRAM node is preferred node */
+ if (!node_isset(preferred_nid, def_alloc_nodemask))
+ /*
+ * With MPOL_PREFERRED policy, once PMEM is allowed,
+ * can falback to all memory nodes.
+ */
+ ac->nodemask = &node_states[N_MEMORY];
+ else
+ ac->nodemask = &def_alloc_nodemask;
+ } else
+ ac->nodemask = nodemask;
+
if (cpusets_enabled()) {
*alloc_mask |= __GFP_HARDWALL;
- if (!ac->nodemask)
+ if (nodes_equal(*ac->nodemask, def_alloc_nodemask))
ac->nodemask = &cpuset_current_mems_allowed;
else
*alloc_flags |= ALLOC_CPUSET;
--
1.8.3.1
