Ravi Jonnalagadda <ravis.opensrc@xxxxxxxxxx> writes:
> From: Srinivasulu Thanneeru <sthanneeru@xxxxxxxxxx>
>
> Existing interleave policy spreads out pages evenly across a set of
> specified nodes, i.e. 1:1 interleave. Upcoming tiered memory systems
> have CPU-less memory nodes with different peak bandwidth and
> latency-bandwidth characteristics. In such systems, we will want to
> use the additional bandwidth provided by lowtier memory for
> bandwidth-intensive applications. However, the default 1:1 interleave
> can lead to suboptimal bandwidth distribution.
>
> Introduce an interleave policy for multi-tiers that is based on
> interleave weights, where pages are assigned from nodes of the tier
> based on the tier weight.
>
> For instance, 50:30:20 are the weights of tiers 0, 1, and 3, which
Does this mean we will allocate 50 pages from node 0 before allocating from
node 1? If so, do we need to divide GCD (greatest common divisor)
before using the weight? That is, use 5:3:2?
> leads to a 50%/30%/20% traffic breakdown across the three tiers.
>
> Signed-off-by: Srinivasulu Thanneeru <sthanneeru@xxxxxxxxxx>
> Co-authored-by: Ravi Jonnalagadda <ravis.opensrc@xxxxxxxxxx>
> ---
> include/linux/memory-tiers.h | 25 +++++++-
> include/linux/sched.h | 2 +
> mm/memory-tiers.c | 31 ++--------
> mm/mempolicy.c | 107 +++++++++++++++++++++++++++++++++--
> 4 files changed, 132 insertions(+), 33 deletions(-)
>
> diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h
> index c62d286749d0..74be39cb56c4 100644
> --- a/include/linux/memory-tiers.h
> +++ b/include/linux/memory-tiers.h
> @@ -2,6 +2,7 @@
> #ifndef _LINUX_MEMORY_TIERS_H
> #define _LINUX_MEMORY_TIERS_H
>
> +#include <linux/device.h>
> #include <linux/types.h>
> #include <linux/nodemask.h>
> #include <linux/kref.h>
> @@ -21,7 +22,27 @@
>
> #define MAX_TIER_INTERLEAVE_WEIGHT 100
>
> -struct memory_tier;
> +struct memory_tier {
> + /* hierarchy of memory tiers */
> + struct list_head list;
> + /* list of all memory types part of this tier */
> + struct list_head memory_types;
> + /*
> + * By default all tiers will have weight as 1, which means they
> + * follow default standard allocation.
> + */
> + unsigned short interleave_weight;
> + /*
> + * start value of abstract distance. memory tier maps
> + * an abstract distance range,
> + * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
> + */
> + int adistance_start;
> + struct device dev;
> + /* All the nodes that are part of all the lower memory tiers. */
> + nodemask_t lower_tier_mask;
> +};
> +
> struct memory_dev_type {
> /* list of memory types that are part of same tier as this type */
> struct list_head tier_sibiling;
> @@ -38,6 +59,8 @@ struct memory_dev_type *alloc_memory_type(int adistance);
> void put_memory_type(struct memory_dev_type *memtype);
> void init_node_memory_type(int node, struct memory_dev_type *default_type);
> void clear_node_memory_type(int node, struct memory_dev_type *memtype);
> +struct memory_tier *node_get_memory_tier(int node);
> +nodemask_t get_memtier_nodemask(struct memory_tier *memtier);
> #ifdef CONFIG_MIGRATION
> int next_demotion_node(int node);
> void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets);
> diff --git a/include/linux/sched.h b/include/linux/sched.h
> index 77f01ac385f7..07ea837c3afb 100644
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -1252,7 +1252,9 @@ struct task_struct {
> /* Protected by alloc_lock: */
> struct mempolicy *mempolicy;
> short il_prev;
> + unsigned short il_count;
> short pref_node_fork;
> + unsigned int current_node;
current_node is only used for interleave? If so, how about follow the
same naming convention, that is, il_XXX?
> #endif
> #ifdef CONFIG_NUMA_BALANCING
> int numa_scan_seq;
> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
> index 7e06c9e0fa41..5e2ddc9f994a 100644
> --- a/mm/memory-tiers.c
> +++ b/mm/memory-tiers.c
> @@ -8,27 +8,6 @@
>
> #include "internal.h"
>
> -struct memory_tier {
> - /* hierarchy of memory tiers */
> - struct list_head list;
> - /* list of all memory types part of this tier */
> - struct list_head memory_types;
> - /*
> - * By default all tiers will have weight as 1, which means they
> - * follow default standard allocation.
> - */
> - unsigned short interleave_weight;
> - /*
> - * start value of abstract distance. memory tier maps
> - * an abstract distance range,
> - * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE
> - */
> - int adistance_start;
> - struct device dev;
> - /* All the nodes that are part of all the lower memory tiers. */
> - nodemask_t lower_tier_mask;
> -};
> -
> struct demotion_nodes {
> nodemask_t preferred;
> };
> @@ -115,7 +94,7 @@ static inline struct memory_tier *to_memory_tier(struct device *device)
> return container_of(device, struct memory_tier, dev);
> }
>
> -static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier)
> +nodemask_t get_memtier_nodemask(struct memory_tier *memtier)
> {
> nodemask_t nodes = NODE_MASK_NONE;
> struct memory_dev_type *memtype;
> @@ -264,7 +243,7 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
> return memtier;
> }
>
> -static struct memory_tier *__node_get_memory_tier(int node)
> +struct memory_tier *node_get_memory_tier(int node)
> {
> pg_data_t *pgdat;
>
> @@ -380,7 +359,7 @@ static void disable_all_demotion_targets(void)
> * We are holding memory_tier_lock, it is safe
> * to access pgda->memtier.
> */
> - memtier = __node_get_memory_tier(node);
> + memtier = node_get_memory_tier(node);
> if (memtier)
> memtier->lower_tier_mask = NODE_MASK_NONE;
> }
> @@ -417,7 +396,7 @@ static void establish_demotion_targets(void)
> best_distance = -1;
> nd = &node_demotion[node];
>
> - memtier = __node_get_memory_tier(node);
> + memtier = node_get_memory_tier(node);
> if (!memtier || list_is_last(&memtier->list, &memory_tiers))
> continue;
> /*
> @@ -562,7 +541,7 @@ static bool clear_node_memory_tier(int node)
> * This also enables us to free the destroyed memory tier
> * with kfree instead of kfree_rcu
> */
> - memtier = __node_get_memory_tier(node);
> + memtier = node_get_memory_tier(node);
> if (memtier) {
> struct memory_dev_type *memtype;
>
> diff --git a/mm/mempolicy.c b/mm/mempolicy.c
> index 42b5567e3773..4f80c6ee1176 100644
> --- a/mm/mempolicy.c
> +++ b/mm/mempolicy.c
> @@ -100,6 +100,8 @@
> #include <linux/ctype.h>
> #include <linux/mm_inline.h>
> #include <linux/mmu_notifier.h>
> +#include <linux/memory-tiers.h>
> +#include <linux/nodemask.h>
> #include <linux/printk.h>
> #include <linux/swapops.h>
>
> @@ -882,8 +884,11 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
>
> old = current->mempolicy;
> current->mempolicy = new;
> - if (new && new->mode == MPOL_INTERLEAVE)
> + if (new && new->mode == MPOL_INTERLEAVE) {
> current->il_prev = MAX_NUMNODES-1;
> + current->il_count = 0;
> + current->current_node = MAX_NUMNODES;
> + }
> task_unlock(current);
> mpol_put(old);
> ret = 0;
> @@ -1899,13 +1904,76 @@ static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd)
> return nd;
> }
>
> +/* Return interleave weight of node from tier's weight */
> +static unsigned short node_interleave_weight(int nid, nodemask_t pol_nodemask)
> +{
> + struct memory_tier *memtier;
> + nodemask_t tier_nodes, tier_and_pol;
> + unsigned short avrg_weight = 0;
> + int node, nnodes, reminder;
> +
> + memtier = node_get_memory_tier(nid);
node_get_memory_tier() must be called with memory_tier_lock held. Or
some other locking mechanism needs to be provided. For example, RCU
read lock.
> +
> + if (!memtier)
> + return 0;
> +
> + tier_nodes = get_memtier_nodemask(memtier);
get_memtier_nodemask() must be called with memory_tier_lock held too.
> + nodes_and(tier_and_pol, tier_nodes, pol_nodemask);
> + nnodes = nodes_weight(tier_and_pol);
> + if (!nnodes)
> + return 0;
> +
> + avrg_weight = memtier->interleave_weight / nnodes;
> + /* Set minimum weight of node as 1 so that at least one page
> + * is allocated.
> + */
> + if (!avrg_weight)
> + return 1;
> +
> + reminder = memtier->interleave_weight % nnodes;
> + if (reminder) {
> + for_each_node_mask(node, tier_and_pol) {
> + /* Increment target node's weight by 1, if it falls
> + * within remaining weightage 'reminder'.
> + */
> + if (node == nid) {
> + if (reminder > 0)
> + avrg_weight = avrg_weight + 1;
> + break;
> + }
> + reminder--;
> + }
> + }
> + return avrg_weight;
> +}
> +
> /* Do dynamic interleaving for a process */
> static unsigned interleave_nodes(struct mempolicy *policy)
> {
> unsigned next;
> struct task_struct *me = current;
> + unsigned short node_weight = 0;
>
> - next = next_node_in(me->il_prev, policy->nodes);
> + /* select current node or next node from nodelist based on
> + * available tier interleave weight.
> + */
> + if (me->current_node == MAX_NUMNODES)
> + next = next_node_in(me->il_prev, policy->nodes);
> + else
> + next = me->current_node;
> + node_weight = node_interleave_weight(next, policy->nodes);
> + if (!node_weight)
> + goto set_il_prev;
> + if (me->il_count < node_weight) {
> + me->il_count++;
> + me->current_node = next;
> + if (me->il_count == node_weight) {
> + me->current_node = MAX_NUMNODES;
> + me->il_count = 0;
> + }
> + }
> +
> +set_il_prev:
> if (next < MAX_NUMNODES)
> me->il_prev = next;
I felt that we can implement the algorithm without introducing
current->current_node (use current->il_prev for it). But I haven't
written the code. So, I may be wrong here.
> return next;
> @@ -1966,9 +2034,10 @@ unsigned int mempolicy_slab_node(void)
> static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)
> {
> nodemask_t nodemask = pol->nodes;
> - unsigned int target, nnodes;
> - int i;
> - int nid;
> + unsigned int target, nnodes, vnnodes = 0;
> + unsigned short node_weight = 0;
> + int nid, vtarget, i;
> +
> /*
> * The barrier will stabilize the nodemask in a register or on
> * the stack so that it will stop changing under the code.
> @@ -1981,7 +2050,33 @@ static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)
> nnodes = nodes_weight(nodemask);
> if (!nnodes)
> return numa_node_id();
> - target = (unsigned int)n % nnodes;
> +
> + /*
> + * Calculate the virtual target for @n in a nodelist that is scaled
> + * with interleave weights....
> + */
> + for_each_node_mask(nid, nodemask) {
> + node_weight = node_interleave_weight(nid, nodemask);
> + if (!node_weight)
> + continue;
> + vnnodes += node_weight;
> + }
> + if (!vnnodes)
> + return numa_node_id();
> + vtarget = (int)((unsigned int)n % vnnodes);
> +
> + /* ...then map it back to the physical nodelist */
> + target = 0;
> + for_each_node_mask(nid, nodemask) {
> + node_weight = node_interleave_weight(nid, nodemask);
> + if (!node_weight)
> + continue;
> + vtarget -= node_weight;
> + if (vtarget < 0)
> + break;
> + target++;
> + }
> +
> nid = first_node(nodemask);
> for (i = 0; i < target; i++)
> nid = next_node(nid, nodemask);
--
Best Regards,
Huang, Ying
