On Mon, Jul 22, 2024 at 01:29:16PM GMT, Zi Yan wrote:
> Instead of open coded check for if memory tiering mode is on and a folio
> is in the top tier memory, use a function to encapsulate the check.
>
> Signed-off-by: Zi Yan <ziy@xxxxxxxxxx>
> Reviewed-by: "Huang, Ying" <ying.huang@xxxxxxxxx>
> ---
> include/linux/mm.h | 6 ++++++
> kernel/sched/fair.c | 3 +--
> mm/huge_memory.c | 6 ++----
> mm/memory-tiers.c | 17 +++++++++++++++++
> mm/memory.c | 3 +--
> mm/mprotect.c | 3 +--
> 6 files changed, 28 insertions(+), 10 deletions(-)
>
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index c227f22ba810..048b2a56d8a3 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1738,6 +1738,8 @@ static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
> __set_bit(pid_bit, &vma->numab_state->pids_active[1]);
> }
> }
> +
> +bool folio_has_cpupid(struct folio *folio);
> #else /* !CONFIG_NUMA_BALANCING */
> static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
> {
> @@ -1791,6 +1793,10 @@ static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
> static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
> {
> }
> +static inline bool folio_has_cpupid(struct folio *folio)
> +{
> + return true;
> +}
> #endif /* CONFIG_NUMA_BALANCING */
>
> #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index 8a5b1ae0aa55..03de808cb3cc 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -1840,8 +1840,7 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,
> * The pages in slow memory node should be migrated according
> * to hot/cold instead of private/shared.
> */
> - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
> - !node_is_toptier(src_nid)) {
> + if (!folio_has_cpupid(folio)) {
> struct pglist_data *pgdat;
> unsigned long rate_limit;
> unsigned int latency, th, def_th;
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index 825317aee88e..d925a93bb9ed 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -1712,8 +1712,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
> * For memory tiering mode, cpupid of slow memory page is used
> * to record page access time. So use default value.
> */
> - if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||
> - node_is_toptier(nid))
> + if (folio_has_cpupid(folio))
> last_cpupid = folio_last_cpupid(folio);
> target_nid = numa_migrate_prep(folio, vmf, haddr, nid, &flags);
> if (target_nid == NUMA_NO_NODE)
> @@ -2066,8 +2065,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
> toptier)
> goto unlock;
>
> - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
> - !toptier)
> + if (!folio_has_cpupid(folio))
> folio_xchg_access_time(folio,
> jiffies_to_msecs(jiffies));
> }
> diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
> index 4775b3a3dabe..7f0360d4e3a0 100644
> --- a/mm/memory-tiers.c
> +++ b/mm/memory-tiers.c
> @@ -6,6 +6,7 @@
> #include <linux/memory.h>
> #include <linux/memory-tiers.h>
> #include <linux/notifier.h>
> +#include <linux/sched/sysctl.h>
>
> #include "internal.h"
>
> @@ -50,6 +51,22 @@ static const struct bus_type memory_tier_subsys = {
> .dev_name = "memory_tier",
> };
>
> +/**
> + * folio_has_cpupid - check if a folio has cpupid information
> + * @folio: folio to check
> + *
> + * folio's _last_cpupid field is repurposed by memory tiering. In memory
> + * tiering mode, cpupid of slow memory folio (not toptier memory) is used to
> + * record page access time.
> + *
> + * Return: the folio _last_cpupid is used as cpupid
> + */
> +bool folio_has_cpupid(struct folio *folio)
> +{
> + return !(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||
> + node_is_toptier(folio_nid(folio));
> +}
> +
The static version of folio_has_cpupid() is defined in include/linux/mm.h
if !CONFIG_NUMA_BALANCING but you define the function in memory-tiers.c
unconditionally, a file that is compiled predicated on CONFIG_NUMA.
So a config with !CONFIG_NUMA_BALANCING but CONFIG_NUMA set results in a
compilation error (I just hit it this morning in mm-unstable).
A minimal fix for this is to wrap the declaration in:
#ifdef CONFIG_NUMA_BALANCING
...
#endif
I've tried this locally and it resolves the issue.
> #ifdef CONFIG_MIGRATION
> static int top_tier_adistance;
> /*
> diff --git a/mm/memory.c b/mm/memory.c
> index 802d0d8a40f9..105e1a0157dd 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -5337,8 +5337,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)
> * For memory tiering mode, cpupid of slow memory page is used
> * to record page access time. So use default value.
> */
> - if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
> - !node_is_toptier(nid))
> + if (!folio_has_cpupid(folio))
> last_cpupid = (-1 & LAST_CPUPID_MASK);
> else
> last_cpupid = folio_last_cpupid(folio);
> diff --git a/mm/mprotect.c b/mm/mprotect.c
> index 222ab434da54..787c3c2bf1b6 100644
> --- a/mm/mprotect.c
> +++ b/mm/mprotect.c
> @@ -161,8 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
> if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
> toptier)
> continue;
> - if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
> - !toptier)
> + if (!folio_has_cpupid(folio))
> folio_xchg_access_time(folio,
> jiffies_to_msecs(jiffies));
> }
> --
> 2.43.0
>
