On 8/9/22 7:28 AM, Huang, Ying wrote:
> "Aneesh Kumar K.V" <aneesh.kumar@xxxxxxxxxxxxx> writes:
>
>> In the current kernel, memory tiers are defined implicitly via a demotion path
>> relationship between NUMA nodes, which is created during the kernel
>> initialization and updated when a NUMA node is hot-added or hot-removed. The
>> current implementation puts all nodes with CPU into the highest tier, and builds
>> the tier hierarchy by establishing the per-node demotion targets based on the
>> distances between nodes.
>>
>> This current memory tier kernel implementation needs to be improved for several
>> important use cases,
>>
>> The current tier initialization code always initializes each memory-only NUMA
>> node into a lower tier. But a memory-only NUMA node may have a high performance
>> memory device (e.g. a DRAM-backed memory-only node on a virtual machine) that
>> should be put into a higher tier.
>>
>> The current tier hierarchy always puts CPU nodes into the top tier. But on a
>> system with HBM or GPU devices, the memory-only NUMA nodes mapping these devices
>> should be in the top tier, and DRAM nodes with CPUs are better to be placed into
>> the next lower tier.
>>
>> With current kernel higher tier node can only be demoted to nodes with shortest
>> distance on the next lower tier as defined by the demotion path, not any other
>> node from any lower tier. This strict, demotion order does not work in all use
>> cases (e.g. some use cases may want to allow cross-socket demotion to another
>> node in the same demotion tier as a fallback when the preferred demotion node is
>> out of space), This demotion order is also inconsistent with the page allocation
>> fallback order when all the nodes in a higher tier are out of space: The page
>> allocation can fall back to any node from any lower tier, whereas the demotion
>> order doesn't allow that.
>>
>> This patch series address the above by defining memory tiers explicitly.
>>
>> Linux kernel presents memory devices as NUMA nodes and each memory device is of
>> a specific type. The memory type of a device is represented by its abstract
>> distance. A memory tier corresponds to a range of abstract distance. This allows
>> for classifying memory devices with a specific performance range into a memory
>> tier.
>>
>> This patch configures the range/chunk size to be 128. The default DRAM abstract
>> distance is 512. We can have 4 memory tiers below the default DRAM with abstract
>> distance range 0 - 127, 127 - 255, 256- 383, 384 - 511. Faster memory devices
>> can be placed in these faster(higher) memory tiers. Slower memory devices like
>> persistent memory will have abstract distance higher than the default DRAM
>> level.
>>
>> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxx>
>> ---
>> include/linux/memory-tiers.h | 15 +++++
>> mm/Makefile | 1 +
>> mm/memory-tiers.c | 107 +++++++++++++++++++++++++++++++++++
>> 3 files changed, 123 insertions(+)
>> create mode 100644 include/linux/memory-tiers.h
>> create mode 100644 mm/memory-tiers.c
>>
>> diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h
>> new file mode 100644
>> index 000000000000..bc7c1b799bef
>> --- /dev/null
>> +++ b/include/linux/memory-tiers.h
>> @@ -0,0 +1,15 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +#ifndef _LINUX_MEMORY_TIERS_H
>> +#define _LINUX_MEMORY_TIERS_H
>> +
>> +/*
>> + * Each tier cover a abstrace distance chunk size of 128
>> + */
>> +#define MEMTIER_CHUNK_BITS 7
>> +#define MEMTIER_CHUNK_SIZE (1 << MEMTIER_CHUNK_BITS)
>> +/*
>> + * Smaller abstract distance value imply faster(higher) memory tiers.
>> + */
>> +#define MEMTIER_ADISTANCE_DRAM (4 * MEMTIER_CHUNK_SIZE)
>> +
>> +#endif /* _LINUX_MEMORY_TIERS_H */
>> diff --git a/mm/Makefile b/mm/Makefile
>> index 6f9ffa968a1a..d30acebc2164 100644
>> --- a/mm/Makefile
>> +++ b/mm/Makefile
>> @@ -92,6 +92,7 @@ obj-$(CONFIG_KFENCE) += kfence/
>> obj-$(CONFIG_FAILSLAB) += failslab.o
>> obj-$(CONFIG_MEMTEST) += memtest.o
>> obj-$(CONFIG_MIGRATION) += migrate.o
>> +obj-$(CONFIG_NUMA) += memory-tiers.o
>> obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o
>> obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
>> obj-$(CONFIG_PAGE_COUNTER) += page_counter.o
>> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
>> new file mode 100644
>> index 000000000000..78b311d9bde9
>> --- /dev/null
>> +++ b/mm/memory-tiers.c
>> @@ -0,0 +1,107 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +#include <linux/types.h>
>> +#include <linux/nodemask.h>
>> +#include <linux/slab.h>
>> +#include <linux/lockdep.h>
>> +#include <linux/memory-tiers.h>
>> +
>> +struct memory_tier {
>> + /* hierarchy of memory tiers */
>> + struct list_head list;
>> + /* list of all memory types part of this tier */
>> + struct list_head memory_types;
>> + /*
>> + * start value of abstract distance. memory tier maps
>> + * an abstract distance range,
>> + * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
>> + */
>> + int adistance_start;
>> +};
>> +
>> +struct memory_dev_type {
>> + /* list of memory types that are part of same tier as this type */
>> + struct list_head tier_sibiling;
>> + /* abstract distance for this specific memory type */
>> + int adistance;
>> + /* Nodes of same abstract distance */
>> + nodemask_t nodes;
>> + struct memory_tier *memtier;
>> +};
>> +
>> +static DEFINE_MUTEX(memory_tier_lock);
>> +static LIST_HEAD(memory_tiers);
>> +static struct memory_dev_type *node_memory_types[MAX_NUMNODES];
>> +/*
>> + * For now let's have 4 memory tier below default DRAM tier.
>> + */
>> +static struct memory_dev_type default_dram_type = {
>> + .adistance = MEMTIER_ADISTANCE_DRAM,
>> + .tier_sibiling = LIST_HEAD_INIT(default_dram_type.tier_sibiling),
>> +};
>> +
>> +static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype)
>> +{
>> + bool found_slot = false;
>> + struct memory_tier *memtier, *new_memtier;
>> + int adistance = memtype->adistance;
>> + unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE;
>> +
>> + lockdep_assert_held_once(&memory_tier_lock);
>> +
>> + /*
>> + * If the memtype is already part of a memory tier,
>> + * just return that.
>> + */
>> + if (memtype->memtier)
>> + return memtype->memtier;
>> +
>> + adistance = round_down(adistance, memtier_adistance_chunk_size);
>> + list_for_each_entry(memtier, &memory_tiers, list) {
>> + if (adistance == memtier->adistance_start) {
>> + memtype->memtier = memtier;
>> + list_add(&memtype->tier_sibiling, &memtier->memory_types);
>> + return memtier;
>> + } else if (adistance < memtier->adistance_start) {
>> + found_slot = true;
>> + break;
>> + }
>> + }
>> +
>> + new_memtier = kmalloc(sizeof(struct memory_tier), GFP_KERNEL);
>> + if (!new_memtier)
>> + return ERR_PTR(-ENOMEM);
>> +
>> + new_memtier->adistance_start = adistance;
>> + INIT_LIST_HEAD(&new_memtier->list);
>> + INIT_LIST_HEAD(&new_memtier->memory_types);
>> + if (found_slot)
>> + list_add_tail(&new_memtier->list, &memtier->list);
>> + else
>> + list_add_tail(&new_memtier->list, &memory_tiers);
>> + memtype->memtier = new_memtier;
>> + list_add(&memtype->tier_sibiling, &new_memtier->memory_types);
>> + return new_memtier;
>> +}
>> +
>> +static int __init memory_tier_init(void)
>> +{
>> + int node;
>> + struct memory_tier *memtier;
>> +
>> + mutex_lock(&memory_tier_lock);
>> + /* CPU only nodes are not part of memory tiers. */
>> + default_dram_type.nodes = node_states[N_MEMORY];
>> +
>> + memtier = find_create_memory_tier(&default_dram_type);
>> + if (IS_ERR(memtier))
>> + panic("%s() failed to register memory tier: %ld\n",
>> + __func__, PTR_ERR(memtier));
>> +
>> + for_each_node_state(node, N_MEMORY)
>> + node_memory_types[node] = &default_dram_type;
>
> Although not absolutely necessary, it seems better to set
> node_memory_types[] before adding nodes to the memory type and adding
> the memory type to the memory tier.
>
>
updated.
-aneesh
