"Aneesh Kumar K.V" <aneesh.kumar@xxxxxxxxxxxxx> writes:
> Also update different helpes to use NODE_DATA()->memtier. Since
> node specific memtier can change based on the reassignment of
> NUMA node to a different memory tiers, accessing NODE_DATA()->memtier
> needs to happen under an rcu read lock or memory_tier_lock.
>
> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@xxxxxxxxxxxxx>
> ---
> include/linux/mmzone.h | 3 +++
> mm/memory-tiers.c | 38 ++++++++++++++++++++++++++++++++------
> 2 files changed, 35 insertions(+), 6 deletions(-)
>
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index aab70355d64f..353812495a70 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -928,6 +928,9 @@ typedef struct pglist_data {
> /* Per-node vmstats */
> struct per_cpu_nodestat __percpu *per_cpu_nodestats;
> atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS];
> +#ifdef CONFIG_NUMA
> + struct memory_tier __rcu *memtier;
> +#endif
> } pg_data_t;
>
> #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
> index 02e514e87d5c..3778ac6a44a1 100644
> --- a/mm/memory-tiers.c
> +++ b/mm/memory-tiers.c
> @@ -5,6 +5,7 @@
> #include <linux/kobject.h>
> #include <linux/memory.h>
> #include <linux/random.h>
> +#include <linux/mmzone.h>
> #include <linux/memory-tiers.h>
>
> #include "internal.h"
> @@ -137,12 +138,18 @@ static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memty
>
> static struct memory_tier *__node_get_memory_tier(int node)
> {
> - struct memory_dev_type *memtype;
> + pg_data_t *pgdat;
>
> - memtype = node_memory_types[node];
> - if (memtype && node_isset(node, memtype->nodes))
> - return memtype->memtier;
> - return NULL;
> + pgdat = NODE_DATA(node);
> + if (!pgdat)
> + return NULL;
> + /*
> + * Since we hold memory_tier_lock, we can avoid
> + * RCU read locks when accessing the details. No
> + * parallel updates are possible here.
> + */
> + return rcu_dereference_check(pgdat->memtier,
> + lockdep_is_held(&memory_tier_lock));
> }
>
> #ifdef CONFIG_MIGRATION
> @@ -295,6 +302,8 @@ static struct memory_tier *set_node_memory_tier(int node)
> {
> struct memory_tier *memtier;
> struct memory_dev_type *memtype;
> + pg_data_t *pgdat = NODE_DATA(node);
> +
>
> lockdep_assert_held_once(&memory_tier_lock);
>
> @@ -307,6 +316,8 @@ static struct memory_tier *set_node_memory_tier(int node)
> memtype = node_memory_types[node];
> node_set(node, memtype->nodes);
> memtier = find_create_memory_tier(memtype);
> + if (!IS_ERR(memtier))
> + rcu_assign_pointer(pgdat->memtier, memtier);
> return memtier;
> }
>
> @@ -319,12 +330,25 @@ static void destroy_memory_tier(struct memory_tier *memtier)
> static bool clear_node_memory_tier(int node)
> {
> bool cleared = false;
> + pg_data_t *pgdat;
> struct memory_tier *memtier;
>
> + pgdat = NODE_DATA(node);
> + if (!pgdat)
> + return false;
> +
> + /*
> + * Make sure that anybody looking at NODE_DATA who finds
> + * a valid memtier finds memory_dev_types with nodes still
> + * linked to the memtier. We achieve this by waiting for
> + * rcu read section to finish using synchronize_rcu.
> + */
The synchronize_rcu() is also needed because we may free the memory
tier.
Best Regards,
Huang, Ying
> memtier = __node_get_memory_tier(node);
> if (memtier) {
> struct memory_dev_type *memtype;
>
> + rcu_assign_pointer(pgdat->memtier, NULL);
> + synchronize_rcu();
> memtype = node_memory_types[node];
> node_clear(node, memtype->nodes);
> if (nodes_empty(memtype->nodes)) {
> @@ -422,8 +446,10 @@ static int __init memory_tier_init(void)
> panic("%s() failed to register memory tier: %ld\n",
> __func__, PTR_ERR(memtier));
>
> - for_each_node_state(node, N_MEMORY)
> + for_each_node_state(node, N_MEMORY) {
> __init_node_memory_type(node, default_dram_type);
> + rcu_assign_pointer(NODE_DATA(node)->memtier, memtier);
> + }
>
> mutex_unlock(&memory_tier_lock);
> #ifdef CONFIG_MIGRATION
