Add following new vmstat events which will track HugeTLB page migration. 1. HUGETLB_MIGRATION_SUCCESS 2. HUGETLB_MIGRATION_FAILURE It follows the existing semantics to accommodate HugeTLB subpages in total page migration statistics. While here, this updates current trace event 'mm_migrate_pages' to accommodate now available HugeTLB based statistics. Cc: Daniel Jordan <daniel.m.jordan@xxxxxxxxxx> Cc: Zi Yan <ziy@xxxxxxxxxx> Cc: John Hubbard <jhubbard@xxxxxxxxxx> Cc: Mike Kravetz <mike.kravetz@xxxxxxxxxx> Cc: Oscar Salvador <osalvador@xxxxxxx> Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx> Cc: linux-mm@xxxxxxxxx Cc: linux-kernel@xxxxxxxxxxxxxxx Signed-off-by: Anshuman Khandual <anshuman.khandual@xxxxxxx> --- Applies on linux-next and v5.9-rc7. Changes in RFC V2: - Added the missing hugetlb_retry in the loop per Oscar - Changed HugeTLB and THP detection sequence per Mike - Changed nr_subpages fetch from compound_nr() instead per Mike Changes in RFC V1: (https://patchwork.kernel.org/patch/11799395/) include/linux/vm_event_item.h | 2 ++ include/trace/events/migrate.h | 13 ++++++--- mm/migrate.c | 48 +++++++++++++++++++++++++++++----- mm/vmstat.c | 2 ++ 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 18e75974d4e3..d1ddad835c19 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -60,6 +60,8 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, THP_MIGRATION_SUCCESS, THP_MIGRATION_FAIL, THP_MIGRATION_SPLIT, + HUGETLB_MIGRATION_SUCCESS, + HUGETLB_MIGRATION_FAIL, #endif #ifdef CONFIG_COMPACTION COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED, diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index 4d434398d64d..f8ffb8aece48 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h @@ -47,10 +47,11 @@ TRACE_EVENT(mm_migrate_pages, TP_PROTO(unsigned long succeeded, unsigned long failed, unsigned long thp_succeeded, unsigned long thp_failed, - unsigned long thp_split, enum migrate_mode mode, int reason), + unsigned long thp_split, unsigned long hugetlb_succeeded, + unsigned long hugetlb_failed, enum migrate_mode mode, int reason), TP_ARGS(succeeded, failed, thp_succeeded, thp_failed, - thp_split, mode, reason), + thp_split, hugetlb_succeeded, hugetlb_failed, mode, reason), TP_STRUCT__entry( __field( unsigned long, succeeded) @@ -58,6 +59,8 @@ TRACE_EVENT(mm_migrate_pages, __field( unsigned long, thp_succeeded) __field( unsigned long, thp_failed) __field( unsigned long, thp_split) + __field( unsigned long, hugetlb_succeeded) + __field( unsigned long, hugetlb_failed) __field( enum migrate_mode, mode) __field( int, reason) ), @@ -68,16 +71,20 @@ TRACE_EVENT(mm_migrate_pages, __entry->thp_succeeded = thp_succeeded; __entry->thp_failed = thp_failed; __entry->thp_split = thp_split; + __entry->hugetlb_succeeded = hugetlb_succeeded; + __entry->hugetlb_failed = hugetlb_failed; __entry->mode = mode; __entry->reason = reason; ), - TP_printk("nr_succeeded=%lu nr_failed=%lu nr_thp_succeeded=%lu nr_thp_failed=%lu nr_thp_split=%lu mode=%s reason=%s", + TP_printk("nr_succeeded=%lu nr_failed=%lu nr_thp_succeeded=%lu nr_thp_failed=%lu nr_thp_split=%lu nr_hugetlb_succeeded=%lu nr_hugetlb_failed=%lu mode=%s reason=%s", __entry->succeeded, __entry->failed, __entry->thp_succeeded, __entry->thp_failed, __entry->thp_split, + __entry->hugetlb_succeeded, + __entry->hugetlb_failed, __print_symbolic(__entry->mode, MIGRATE_MODE), __print_symbolic(__entry->reason, MIGRATE_REASON)) ); diff --git a/mm/migrate.c b/mm/migrate.c index 5ca5842df5db..0aac9d39778c 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1415,13 +1415,17 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, { int retry = 1; int thp_retry = 1; + int hugetlb_retry = 1; int nr_failed = 0; int nr_succeeded = 0; int nr_thp_succeeded = 0; int nr_thp_failed = 0; int nr_thp_split = 0; + int nr_hugetlb_succeeded = 0; + int nr_hugetlb_failed = 0; int pass = 0; bool is_thp = false; + bool is_hugetlb = false; struct page *page; struct page *page2; int swapwrite = current->flags & PF_SWAPWRITE; @@ -1430,9 +1434,10 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, if (!swapwrite) current->flags |= PF_SWAPWRITE; - for (pass = 0; pass < 10 && (retry || thp_retry); pass++) { + for (pass = 0; pass < 10 && (retry || thp_retry || hugetlb_retry); pass++) { retry = 0; thp_retry = 0; + hugetlb_retry = 0; list_for_each_entry_safe(page, page2, from, lru) { retry: @@ -1441,11 +1446,19 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, * Capture required information that might get lost * during migration. */ - is_thp = PageTransHuge(page) && !PageHuge(page); - nr_subpages = thp_nr_pages(page); + is_thp = false; + is_hugetlb = false; + if (PageTransHuge(page)) { + if (PageHuge(page)) + is_hugetlb = true; + else + is_thp = true; + } + nr_subpages = compound_nr(page); + cond_resched(); - if (PageHuge(page)) + if (is_hugetlb) rc = unmap_and_move_huge_page(get_new_page, put_new_page, private, page, pass > 2, mode, reason); @@ -1481,6 +1494,11 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, nr_failed += nr_subpages; goto out; } + if (is_hugetlb) { + nr_hugetlb_failed++; + nr_failed += nr_subpages; + goto out; + } nr_failed++; goto out; case -EAGAIN: @@ -1488,6 +1506,10 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, thp_retry++; break; } + if (is_hugetlb) { + hugetlb_retry++; + break; + } retry++; break; case MIGRATEPAGE_SUCCESS: @@ -1496,6 +1518,11 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, nr_succeeded += nr_subpages; break; } + if (is_hugetlb) { + nr_hugetlb_succeeded++; + nr_succeeded += nr_subpages; + break; + } nr_succeeded++; break; default: @@ -1510,13 +1537,19 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, nr_failed += nr_subpages; break; } + if (is_hugetlb) { + nr_hugetlb_failed++; + nr_failed += nr_subpages; + break; + } nr_failed++; break; } } } - nr_failed += retry + thp_retry; + nr_failed += retry + thp_retry + hugetlb_retry; nr_thp_failed += thp_retry; + nr_hugetlb_failed += hugetlb_retry; rc = nr_failed; out: count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); @@ -1524,8 +1557,11 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded); count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed); count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split); + count_vm_events(HUGETLB_MIGRATION_SUCCESS, nr_hugetlb_succeeded); + count_vm_events(HUGETLB_MIGRATION_FAIL, nr_hugetlb_failed); trace_mm_migrate_pages(nr_succeeded, nr_failed, nr_thp_succeeded, - nr_thp_failed, nr_thp_split, mode, reason); + nr_thp_failed, nr_thp_split, nr_hugetlb_succeeded, + nr_hugetlb_failed, mode, reason); if (!swapwrite) current->flags &= ~PF_SWAPWRITE; diff --git a/mm/vmstat.c b/mm/vmstat.c index 79e5cd0abd0e..12fd35ba135f 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1286,6 +1286,8 @@ const char * const vmstat_text[] = { "thp_migration_success", "thp_migration_fail", "thp_migration_split", + "hugetlb_migration_success", + "hugetlb_migration_fail", #endif #ifdef CONFIG_COMPACTION "compact_migrate_scanned", -- 2.20.1
