With explicit metadata page management support, it's important to know if
the source page for migration has metadata associated with it for two
reasons:
- The page allocator knows to skip metadata pages (which cannot have
metadata) when allocating the destination page.
- The associated metadata page is correctly reserved when fulfilling the
allocation for the destination page.
When choosing the destination during migration, keep track if the source
page has metadata.
The mbind() system call changes the NUMA allocation policy for the
specified memory range and nodemask. If the MPOL_MF_MOVE or
MPOL_MF_MOVE_ALL flags are set, then any existing allocations that fall
within the range which don't conform to the specified policy will be
migrated. The function that allocates the destination page for migration
is new_page(), teach it too about source pages with metadata.
Signed-off-by: Alexandru Elisei <alexandru.elisei@xxxxxxx>
---
arch/arm64/include/asm/memory_metadata.h | 4 ++++
include/asm-generic/memory_metadata.h | 4 ++++
mm/mempolicy.c | 4 ++++
mm/migrate.c | 6 ++++++
4 files changed, 18 insertions(+)
diff --git a/arch/arm64/include/asm/memory_metadata.h b/arch/arm64/include/asm/memory_metadata.h
index c57c435c8ba3..132707fce9ab 100644
--- a/arch/arm64/include/asm/memory_metadata.h
+++ b/arch/arm64/include/asm/memory_metadata.h
@@ -21,6 +21,10 @@ static inline bool alloc_can_use_metadata_pages(gfp_t gfp_mask)
#define page_has_metadata(page) page_mte_tagged(page)
+static inline bool folio_has_metadata(struct folio *folio)
+{
+ return page_has_metadata(&folio->page);
+}
#endif /* CONFIG_MEMORY_METADATA */
#endif /* __ASM_MEMORY_METADATA_H */
diff --git a/include/asm-generic/memory_metadata.h b/include/asm-generic/memory_metadata.h
index 02b279823920..8f4e2fba222f 100644
--- a/include/asm-generic/memory_metadata.h
+++ b/include/asm-generic/memory_metadata.h
@@ -20,6 +20,10 @@ static inline bool page_has_metadata(struct page *page)
{
return false;
}
+static inline bool folio_has_metadata(struct folio *folio)
+{
+ return false;
+}
#endif /* !CONFIG_MEMORY_METADATA */
#endif /* __ASM_GENERIC_MEMORY_METADATA_H */
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index edc25195f5bd..d164b5c50243 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -103,6 +103,7 @@
#include <linux/printk.h>
#include <linux/swapops.h>
+#include <asm/memory_metadata.h>
#include <asm/tlbflush.h>
#include <asm/tlb.h>
#include <linux/uaccess.h>
@@ -1219,6 +1220,9 @@ static struct folio *new_folio(struct folio *src, unsigned long start)
if (folio_test_large(src))
gfp = GFP_TRANSHUGE;
+ if (folio_has_metadata(src))
+ gfp |= __GFP_TAGGED;
+
/*
* if !vma, vma_alloc_folio() will use task or system default policy
*/
diff --git a/mm/migrate.c b/mm/migrate.c
index 24baad2571e3..c6826562220a 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -51,6 +51,7 @@
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
+#include <asm/memory_metadata.h>
#include <asm/tlbflush.h>
#include <trace/events/migrate.h>
@@ -1990,6 +1991,9 @@ struct folio *alloc_migration_target(struct folio *src, unsigned long private)
if (nid == NUMA_NO_NODE)
nid = folio_nid(src);
+ if (folio_has_metadata(src))
+ gfp_mask |= __GFP_TAGGED;
+
if (folio_test_hugetlb(src)) {
struct hstate *h = folio_hstate(src);
@@ -2476,6 +2480,8 @@ static struct folio *alloc_misplaced_dst_folio(struct folio *src,
__GFP_NOWARN;
gfp &= ~__GFP_RECLAIM;
}
+ if (folio_has_metadata(src))
+ gfp |= __GFP_TAGGED;
return __folio_alloc_node(gfp, order, nid);
}
--
2.41.0
