From: Hyeongtak Ji <hyeongtak.ji@xxxxxx>
This patch introduces DAMOS_PROMOTE action for paddr mode.
It includes renaming alloc_demote_folio to alloc_migrate_folio to use it
for promotion as well.
Signed-off-by: Hyeongtak Ji <hyeongtak.ji@xxxxxx>
Signed-off-by: Honggyu Kim <honggyu.kim@xxxxxx>
---
include/linux/damon.h | 2 ++
include/linux/migrate_mode.h | 1 +
include/linux/vm_event_item.h | 1 +
include/trace/events/migrate.h | 3 ++-
mm/damon/paddr.c | 45 ++++++++++++++++++++++++++++------
mm/damon/sysfs-schemes.c | 1 +
mm/vmstat.c | 1 +
7 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/include/linux/damon.h b/include/linux/damon.h
index 86e66772766b..d7e52d0228b4 100644
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -105,6 +105,7 @@ struct damon_target {
* @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE.
* @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists.
* @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists.
+ * @DAMOS_PROMOTE: Do promotion for the given region.
* @DAMOS_DEMOTE: Do demotion for the given region.
* @DAMOS_STAT: Do nothing but count the stat.
* @NR_DAMOS_ACTIONS: Total number of DAMOS actions
@@ -123,6 +124,7 @@ enum damos_action {
DAMOS_NOHUGEPAGE,
DAMOS_LRU_PRIO,
DAMOS_LRU_DEPRIO,
+ DAMOS_PROMOTE,
DAMOS_DEMOTE,
DAMOS_STAT, /* Do nothing but only record the stat */
NR_DAMOS_ACTIONS,
diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h
index f37cc03f9369..63f75eb9abf3 100644
--- a/include/linux/migrate_mode.h
+++ b/include/linux/migrate_mode.h
@@ -29,6 +29,7 @@ enum migrate_reason {
MR_CONTIG_RANGE,
MR_LONGTERM_PIN,
MR_DEMOTION,
+ MR_PROMOTION,
MR_TYPES
};
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 8abfa1240040..63cf920afeaa 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -44,6 +44,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
PGDEMOTE_KSWAPD,
PGDEMOTE_DIRECT,
PGDEMOTE_KHUGEPAGED,
+ PGPROMOTE,
PGSCAN_KSWAPD,
PGSCAN_DIRECT,
PGSCAN_KHUGEPAGED,
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index 0190ef725b43..f0dd569c1e62 100644
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -22,7 +22,8 @@
EM( MR_NUMA_MISPLACED, "numa_misplaced") \
EM( MR_CONTIG_RANGE, "contig_range") \
EM( MR_LONGTERM_PIN, "longterm_pin") \
- EMe(MR_DEMOTION, "demotion")
+ EM( MR_DEMOTION, "demotion") \
+ EMe(MR_PROMOTION, "promotion")
/*
* First define the enums in the above macros to be exported to userspace
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index 23e37ce57202..37a7b34a36dd 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -229,6 +229,7 @@ static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio)
enum migration_mode {
MIG_PAGEOUT,
+ MIG_PROMOTE,
MIG_DEMOTE,
};
@@ -241,9 +242,26 @@ static unsigned int migrate_folio_list(struct list_head *migrate_folios,
struct pglist_data *pgdat,
enum migration_mode mm)
{
- int target_nid = next_demotion_node(pgdat->node_id);
+ int target_nid;
unsigned int nr_succeeded;
nodemask_t allowed_mask;
+ int reason;
+ enum vm_event_item vm_event;
+
+ switch (mm) {
+ case MIG_PROMOTE:
+ target_nid = next_promotion_node(pgdat->node_id);
+ reason = MR_PROMOTION;
+ vm_event = PGPROMOTE;
+ break;
+ case MIG_DEMOTE:
+ target_nid = next_demotion_node(pgdat->node_id);
+ reason = MR_DEMOTION;
+ vm_event = PGDEMOTE_DIRECT;
+ break;
+ default:
+ return 0;
+ }
struct migration_target_control mtc = {
/*
@@ -263,14 +281,19 @@ static unsigned int migrate_folio_list(struct list_head *migrate_folios,
if (list_empty(migrate_folios))
return 0;
- node_get_allowed_targets(pgdat, &allowed_mask);
+ if (mm == MIG_DEMOTE) {
+ node_get_allowed_targets(pgdat, &allowed_mask);
+ } else if (mm == MIG_PROMOTE) {
+ /* TODO: Need to add upper_tier_mask at struct memory_tier. */
+ allowed_mask = NODE_MASK_NONE;
+ }
/* Migration ignores all cpuset and mempolicy settings */
migrate_pages(migrate_folios, alloc_migrate_folio, NULL,
- (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
+ (unsigned long)&mtc, MIGRATE_ASYNC, reason,
&nr_succeeded);
- __count_vm_events(PGDEMOTE_DIRECT, nr_succeeded);
+ __count_vm_events(vm_event, nr_succeeded);
return nr_succeeded;
}
@@ -359,7 +382,8 @@ static unsigned int damon_pa_migrate_folio_list(struct list_head *folio_list,
VM_BUG_ON_FOLIO(folio_test_active(folio), folio);
references = folio_check_references(folio);
- if (references == FOLIOREF_KEEP)
+ if (references == FOLIOREF_KEEP ||
+ (references == FOLIOREF_RECLAIM && mm == MIG_PROMOTE))
goto keep_locked;
/* Relocate its contents to another node. */
@@ -452,8 +476,10 @@ static unsigned long damon_pa_migrate(struct damon_region *r, struct damos *s,
if (damos_pa_filter_out(s, folio))
goto put_folio;
- folio_clear_referenced(folio);
- folio_test_clear_young(folio);
+ if (mm != MIG_PROMOTE) {
+ folio_clear_referenced(folio);
+ folio_test_clear_young(folio);
+ }
if (!folio_isolate_lru(folio))
goto put_folio;
/*
@@ -471,6 +497,7 @@ static unsigned long damon_pa_migrate(struct damon_region *r, struct damos *s,
case MIG_PAGEOUT:
applied = reclaim_pages(&folio_list);
break;
+ case MIG_PROMOTE:
case MIG_DEMOTE:
applied = damon_pa_migrate_pages(&folio_list, mm);
break;
@@ -530,6 +557,8 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
return damon_pa_mark_accessed(r, scheme);
case DAMOS_LRU_DEPRIO:
return damon_pa_deactivate_pages(r, scheme);
+ case DAMOS_PROMOTE:
+ return damon_pa_migrate(r, scheme, MIG_PROMOTE);
case DAMOS_DEMOTE:
return damon_pa_migrate(r, scheme, MIG_DEMOTE);
case DAMOS_STAT:
@@ -552,6 +581,8 @@ static int damon_pa_scheme_score(struct damon_ctx *context,
return damon_hot_score(context, r, scheme);
case DAMOS_LRU_DEPRIO:
return damon_cold_score(context, r, scheme);
+ case DAMOS_PROMOTE:
+ return damon_hot_score(context, r, scheme);
case DAMOS_DEMOTE:
return damon_cold_score(context, r, scheme);
default:
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index 53e47fad5021..9bc48932eb6c 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -1186,6 +1186,7 @@ static const char * const damon_sysfs_damos_action_strs[] = {
"nohugepage",
"lru_prio",
"lru_deprio",
+ "promote",
"demote",
"stat",
};
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 359460deb377..c703abdb8137 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1282,6 +1282,7 @@ const char * const vmstat_text[] = {
"pgdemote_kswapd",
"pgdemote_direct",
"pgdemote_khugepaged",
+ "pgpromote",
"pgscan_kswapd",
"pgscan_direct",
"pgscan_khugepaged",
--
2.34.1
