The patch titled
Subject: memory tiering: introduce folio_has_cpupid() check
has been added to the -mm mm-unstable branch. Its filename is
memory-tiering-introduce-folio_has_cpupid-check.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/memory-tiering-introduce-folio_has_cpupid-check.patch
This patch will later appear in the mm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via the mm-everything
branch at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there every 2-3 working days
------------------------------------------------------
From: Zi Yan <ziy@xxxxxxxxxx>
Subject: memory tiering: introduce folio_has_cpupid() check
Date: Mon, 22 Jul 2024 13:29:16 -0400
Instead of open coded check for if memory tiering mode is on and a folio
is in the top tier memory, use a function to encapsulate the check.
Link: https://lkml.kernel.org/r/20240722172917.503370-2-ziy@xxxxxxxxxx
Signed-off-by: Zi Yan <ziy@xxxxxxxxxx>
Reviewed-by: "Huang, Ying" <ying.huang@xxxxxxxxx>
Cc: Baolin Wang <baolin.wang@xxxxxxxxxxxxxxxxx>
Cc: David Hildenbrand <david@xxxxxxxxxx>
Cc: Kefeng Wang <wangkefeng.wang@xxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
include/linux/mm.h | 6 ++++++
kernel/sched/fair.c | 3 +--
mm/huge_memory.c | 6 ++----
mm/memory-tiers.c | 17 +++++++++++++++++
mm/memory.c | 3 +--
mm/mprotect.c | 3 +--
6 files changed, 28 insertions(+), 10 deletions(-)
--- a/include/linux/mm.h~memory-tiering-introduce-folio_has_cpupid-check
+++ a/include/linux/mm.h
@@ -1738,6 +1738,8 @@ static inline void vma_set_access_pid_bi
__set_bit(pid_bit, &vma->numab_state->pids_active[1]);
}
}
+
+bool folio_has_cpupid(struct folio *folio);
#else /* !CONFIG_NUMA_BALANCING */
static inline int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
{
@@ -1791,6 +1793,10 @@ static inline bool cpupid_match_pid(stru
static inline void vma_set_access_pid_bit(struct vm_area_struct *vma)
{
}
+static inline bool folio_has_cpupid(struct folio *folio)
+{
+ return true;
+}
#endif /* CONFIG_NUMA_BALANCING */
#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
--- a/kernel/sched/fair.c~memory-tiering-introduce-folio_has_cpupid-check
+++ a/kernel/sched/fair.c
@@ -1840,8 +1840,7 @@ bool should_numa_migrate_memory(struct t
* The pages in slow memory node should be migrated according
* to hot/cold instead of private/shared.
*/
- if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
- !node_is_toptier(src_nid)) {
+ if (!folio_has_cpupid(folio)) {
struct pglist_data *pgdat;
unsigned long rate_limit;
unsigned int latency, th, def_th;
--- a/mm/huge_memory.c~memory-tiering-introduce-folio_has_cpupid-check
+++ a/mm/huge_memory.c
@@ -1712,8 +1712,7 @@ vm_fault_t do_huge_pmd_numa_page(struct
* For memory tiering mode, cpupid of slow memory page is used
* to record page access time. So use default value.
*/
- if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||
- node_is_toptier(nid))
+ if (folio_has_cpupid(folio))
last_cpupid = folio_last_cpupid(folio);
target_nid = numa_migrate_prep(folio, vmf, haddr, nid, &flags);
if (target_nid == NUMA_NO_NODE)
@@ -2066,8 +2065,7 @@ int change_huge_pmd(struct mmu_gather *t
toptier)
goto unlock;
- if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
- !toptier)
+ if (!folio_has_cpupid(folio))
folio_xchg_access_time(folio,
jiffies_to_msecs(jiffies));
}
--- a/mm/memory.c~memory-tiering-introduce-folio_has_cpupid-check
+++ a/mm/memory.c
@@ -5337,8 +5337,7 @@ static vm_fault_t do_numa_page(struct vm
* For memory tiering mode, cpupid of slow memory page is used
* to record page access time. So use default value.
*/
- if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
- !node_is_toptier(nid))
+ if (!folio_has_cpupid(folio))
last_cpupid = (-1 & LAST_CPUPID_MASK);
else
last_cpupid = folio_last_cpupid(folio);
--- a/mm/memory-tiers.c~memory-tiering-introduce-folio_has_cpupid-check
+++ a/mm/memory-tiers.c
@@ -6,6 +6,7 @@
#include <linux/memory.h>
#include <linux/memory-tiers.h>
#include <linux/notifier.h>
+#include <linux/sched/sysctl.h>
#include "internal.h"
@@ -50,6 +51,22 @@ static const struct bus_type memory_tier
.dev_name = "memory_tier",
};
+/**
+ * folio_has_cpupid - check if a folio has cpupid information
+ * @folio: folio to check
+ *
+ * folio's _last_cpupid field is repurposed by memory tiering. In memory
+ * tiering mode, cpupid of slow memory folio (not toptier memory) is used to
+ * record page access time.
+ *
+ * Return: the folio _last_cpupid is used as cpupid
+ */
+bool folio_has_cpupid(struct folio *folio)
+{
+ return !(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) ||
+ node_is_toptier(folio_nid(folio));
+}
+
#ifdef CONFIG_MIGRATION
static int top_tier_adistance;
/*
--- a/mm/mprotect.c~memory-tiering-introduce-folio_has_cpupid-check
+++ a/mm/mprotect.c
@@ -161,8 +161,7 @@ static long change_pte_range(struct mmu_
if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
toptier)
continue;
- if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
- !toptier)
+ if (!folio_has_cpupid(folio))
folio_xchg_access_time(folio,
jiffies_to_msecs(jiffies));
}
_
Patches currently in -mm which might be from ziy@xxxxxxxxxx are
memory-tiering-read-last_cpupid-correctly-in-do_huge_pmd_numa_page.patch
memory-tiering-introduce-folio_has_cpupid-check.patch
memory-tiering-count-pgpromote_success-when-mem-tiering-is-enabled.patch
