Baolin Wang <baolin.wang@xxxxxxxxxxxxxxxxx> writes:
> On 11/4/2022 4:30 PM, Huang Ying wrote:
>> Quite straightforward, the page functions are converted to
>> corresponding folio functions. Same for comments.
>> THP specific code are converted to be large folio.
>> Signed-off-by: "Huang, Ying" <ying.huang@xxxxxxxxx>
>> Cc: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
>> Cc: Zi Yan <ziy@xxxxxxxxxx>
>> Cc: Yang Shi <shy828301@xxxxxxxxx>
>> Cc: Baolin Wang <baolin.wang@xxxxxxxxxxxxxxxxx>
>> Cc: Oscar Salvador <osalvador@xxxxxxx>
>> Cc: Matthew Wilcox <willy@xxxxxxxxxxxxx>
>> ---
>> mm/migrate.c | 201 +++++++++++++++++++++++++++------------------------
>> 1 file changed, 107 insertions(+), 94 deletions(-)
>> diff --git a/mm/migrate.c b/mm/migrate.c
>> index f6dd749dd2f8..b41289ef3b65 100644
>> --- a/mm/migrate.c
>> +++ b/mm/migrate.c
>> @@ -1373,218 +1373,231 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
>> return rc;
>> }
>> -static inline int try_split_thp(struct page *page, struct
>> list_head *split_pages)
>> +static inline int try_split_folio(struct folio *folio, struct list_head *split_folios)
>> {
>> int rc;
>> - lock_page(page);
>> - rc = split_huge_page_to_list(page, split_pages);
>> - unlock_page(page);
>> + folio_lock(folio);
>> + rc = split_folio_to_list(folio, split_folios);
>> + folio_unlock(folio);
>> if (!rc)
>> - list_move_tail(&page->lru, split_pages);
>> + list_move_tail(&folio->lru, split_folios);
>> return rc;
>> }
>> /*
>> - * migrate_pages - migrate the pages specified in a list, to the free pages
>> + * migrate_pages - migrate the folios specified in a list, to the free folios
>> * supplied as the target for the page migration
>> *
>> - * @from: The list of pages to be migrated.
>> - * @get_new_page: The function used to allocate free pages to be used
>> - * as the target of the page migration.
>> - * @put_new_page: The function used to free target pages if migration
>> + * @from: The list of folios to be migrated.
>> + * @get_new_page: The function used to allocate free folios to be used
>> + * as the target of the folio migration.
>> + * @put_new_page: The function used to free target folios if migration
>> * fails, or NULL if no special handling is necessary.
>> * @private: Private data to be passed on to get_new_page()
>> * @mode: The migration mode that specifies the constraints for
>> - * page migration, if any.
>> - * @reason: The reason for page migration.
>> - * @ret_succeeded: Set to the number of normal pages migrated successfully if
>> + * folio migration, if any.
>> + * @reason: The reason for folio migration.
>> + * @ret_succeeded: Set to the number of folios migrated successfully if
>> * the caller passes a non-NULL pointer.
>> *
>> - * The function returns after 10 attempts or if no pages are movable any more
>> - * because the list has become empty or no retryable pages exist any more.
>> - * It is caller's responsibility to call putback_movable_pages() to return pages
>> + * The function returns after 10 attempts or if no folios are movable any more
>> + * because the list has become empty or no retryable folios exist any more.
>> + * It is caller's responsibility to call putback_movable_pages() to return folios
>> * to the LRU or free list only if ret != 0.
>> *
>> - * Returns the number of {normal page, THP, hugetlb} that were not migrated, or
>> - * an error code. The number of THP splits will be considered as the number of
>> - * non-migrated THP, no matter how many subpages of the THP are migrated successfully.
>> + * Returns the number of {normal folio, large folio, hugetlb} that were not
>> + * migrated, or an error code. The number of large folio splits will be
>> + * considered as the number of non-migrated large folio, no matter how many
>> + * split folios of the large folio are migrated successfully.
>> */
>> int migrate_pages(struct list_head *from, new_page_t get_new_page,
>> free_page_t put_new_page, unsigned long private,
>> enum migrate_mode mode, int reason, unsigned int *ret_succeeded)
>> {
>> int retry = 1;
>> + int large_retry = 1;
>> int thp_retry = 1;
>> int nr_failed = 0;
>> int nr_failed_pages = 0;
>> int nr_retry_pages = 0;
>> int nr_succeeded = 0;
>> int nr_thp_succeeded = 0;
>> + int nr_large_failed = 0;
>> int nr_thp_failed = 0;
>> int nr_thp_split = 0;
>> int pass = 0;
>> + bool is_large = false;
>> bool is_thp = false;
>> - struct page *page;
>> - struct page *page2;
>> - int rc, nr_subpages;
>> - LIST_HEAD(ret_pages);
>> - LIST_HEAD(thp_split_pages);
>> + struct folio *folio, *folio2;
>> + int rc, nr_pages;
>> + LIST_HEAD(ret_folios);
>> + LIST_HEAD(split_folios);
>> bool nosplit = (reason == MR_NUMA_MISPLACED);
>> - bool no_subpage_counting = false;
>> + bool no_split_folio_counting = false;
>> trace_mm_migrate_pages_start(mode, reason);
>> -thp_subpage_migration:
>> - for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
>> +split_folio_migration:
>> + for (pass = 0; pass < 10 && (retry || large_retry); pass++) {
>> retry = 0;
>> + large_retry = 0;
>> thp_retry = 0;
>> nr_retry_pages = 0;
>> - list_for_each_entry_safe(page, page2, from, lru) {
>> + list_for_each_entry_safe(folio, folio2, from, lru) {
>> /*
>> - * THP statistics is based on the source huge page.
>> - * Capture required information that might get lost
>> - * during migration.
>> + * large folio statistics is based on the source large
>
> Nit: s/large/Large
Thanks. Will change this.
>> + * folio. Capture required information that might get
>> + * lost during migration.
>> */
>> - is_thp = PageTransHuge(page) && !PageHuge(page);
>> - nr_subpages = compound_nr(page);
>> + is_large = folio_test_large(folio) && !folio_test_hugetlb(folio);
>> + is_thp = is_large && folio_test_pmd_mappable(folio);
>> + nr_pages = folio_nr_pages(folio);
>> cond_resched();
>> - if (PageHuge(page))
>> + if (folio_test_hugetlb(folio))
>> rc = unmap_and_move_huge_page(get_new_page,
>> - put_new_page, private, page,
>> - pass > 2, mode, reason,
>> - &ret_pages);
>> + put_new_page, private,
>> + &folio->page, pass > 2, mode,
>> + reason,
>> + &ret_folios);
>> else
>> rc = unmap_and_move(get_new_page, put_new_page,
>> - private, page_folio(page), pass > 2, mode,
>> - reason, &ret_pages);
>> + private, folio, pass > 2, mode,
>> + reason, &ret_folios);
>> /*
>> * The rules are:
>> - * Success: non hugetlb page will be freed, hugetlb
>> - * page will be put back
>> + * Success: non hugetlb folio will be freed, hugetlb
>> + * folio will be put back
>> * -EAGAIN: stay on the from list
>> * -ENOMEM: stay on the from list
>> * -ENOSYS: stay on the from list
>> - * Other errno: put on ret_pages list then splice to
>> + * Other errno: put on ret_folios list then splice to
>> * from list
>> */
>> switch(rc) {
>> /*
>> - * THP migration might be unsupported or the
>> - * allocation could've failed so we should
>> - * retry on the same page with the THP split
>> - * to base pages.
>> + * Large folio migration might be unsupported or
>> + * the allocation could've failed so we should retry
>> + * on the same folio with the large folio split
>> + * to normal folios.
>> *
>> - * Sub-pages are put in thp_split_pages, and
>> + * Split folios are put in split_folios, and
>> * we will migrate them after the rest of the
>> * list is processed.
>> */
>> case -ENOSYS:
>> - /* THP migration is unsupported */
>> - if (is_thp) {
>> - nr_thp_failed++;
>> - if (!try_split_thp(page, &thp_split_pages)) {
>> - nr_thp_split++;
>> + /* Large folio migration is unsupported */
>> + if (is_large) {
>> + nr_large_failed++;
>> + nr_thp_failed += is_thp;
>> + if (!try_split_folio(folio, &split_folios)) {
>> + nr_thp_split += is_thp;
>> break;
>> }
>> /* Hugetlb migration is unsupported */
>> - } else if (!no_subpage_counting) {
>> + } else if (!no_split_folio_counting) {
>> nr_failed++;
>> }
>> - nr_failed_pages += nr_subpages;
>> - list_move_tail(&page->lru, &ret_pages);
>> + nr_failed_pages += nr_pages;
>> + list_move_tail(&folio->lru, &ret_folios);
>> break;
>> case -ENOMEM:
>> /*
>> * When memory is low, don't bother to try to migrate
>> - * other pages, just exit.
>> + * other folios, just exit.
>> */
>> - if (is_thp) {
>> - nr_thp_failed++;
>> - /* THP NUMA faulting doesn't split THP to retry. */
>> - if (!nosplit && !try_split_thp(page, &thp_split_pages)) {
>> - nr_thp_split++;
>> + if (is_large) {
>> + nr_large_failed++;
>> + nr_thp_failed += is_thp;
>> + /* Large folio NUMA faulting doesn't split to retry. */
>> + if (!nosplit && !try_split_folio(folio, &split_folios)) {
>
> I am not sure if need to add a is_thp validation before calling
> try_split_folio()?
IIUC try_split_folio() can deal with large folio with arbitrary order now.
> BTW, you should rebase on the mm-unstable branch, since I've added a
> retry for THP split.
Yes. Andrew remind me too.
Best Regards,
Huang, Ying
>> + nr_thp_split += is_thp;
>> break;
>> }
>> - } else if (!no_subpage_counting) {
>> + } else if (!no_split_folio_counting) {
>> nr_failed++;
>> }
>> - nr_failed_pages += nr_subpages +
>> nr_retry_pages;
>> + nr_failed_pages += nr_pages + nr_retry_pages;
>> /*
>> - * There might be some subpages of fail-to-migrate THPs
>> - * left in thp_split_pages list. Move them back to migration
>> + * There might be some split folios of fail-to-migrate large
>> + * folios left in split_folios list. Move them back to migration
>> * list so that they could be put back to the right list by
>> - * the caller otherwise the page refcnt will be leaked.
>> + * the caller otherwise the folio refcnt will be leaked.
>> */
>> - list_splice_init(&thp_split_pages, from);
>> + list_splice_init(&split_folios, from);
>> /* nr_failed isn't updated for not used */
>> + nr_large_failed += large_retry;
>> nr_thp_failed += thp_retry;
>> goto out;
>> case -EAGAIN:
>> - if (is_thp)
>> - thp_retry++;
>> - else if (!no_subpage_counting)
>> + if (is_large) {
>> + large_retry++;
>> + thp_retry += is_thp;
>> + } else if (!no_split_folio_counting) {
>> retry++;
>> - nr_retry_pages += nr_subpages;
>> + }
>> + nr_retry_pages += nr_pages;
>> break;
>> case MIGRATEPAGE_SUCCESS:
>> - nr_succeeded += nr_subpages;
>> - if (is_thp)
>> - nr_thp_succeeded++;
>> + nr_succeeded += nr_pages;
>> + nr_thp_succeeded += is_thp;
>> break;
>> default:
>> /*
>> * Permanent failure (-EBUSY, etc.):
>> - * unlike -EAGAIN case, the failed page is
>> - * removed from migration page list and not
>> + * unlike -EAGAIN case, the failed folio is
>> + * removed from migration folio list and not
>> * retried in the next outer loop.
>> */
>> - if (is_thp)
>> - nr_thp_failed++;
>> - else if (!no_subpage_counting)
>> + if (is_large) {
>> + nr_large_failed++;
>> + nr_thp_failed += is_thp;
>> + } else if (!no_split_folio_counting) {
>> nr_failed++;
>> + }
>> - nr_failed_pages += nr_subpages;
>> + nr_failed_pages += nr_pages;
>> break;
>> }
>> }
>> }
>> nr_failed += retry;
>> + nr_large_failed += large_retry;
>> nr_thp_failed += thp_retry;
>> nr_failed_pages += nr_retry_pages;
>> /*
>> - * Try to migrate subpages of fail-to-migrate THPs, no nr_failed
>> - * counting in this round, since all subpages of a THP is counted
>> - * as 1 failure in the first round.
>> + * Try to migrate split folios of fail-to-migrate large folios, no
>> + * nr_failed counting in this round, since all split folios of a
>> + * large folio is counted as 1 failure in the first round.
>> */
>> - if (!list_empty(&thp_split_pages)) {
>> + if (!list_empty(&split_folios)) {
>> /*
>> - * Move non-migrated pages (after 10 retries) to ret_pages
>> + * Move non-migrated folios (after 10 retries) to ret_folios
>> * to avoid migrating them again.
>> */
>> - list_splice_init(from, &ret_pages);
>> - list_splice_init(&thp_split_pages, from);
>> - no_subpage_counting = true;
>> + list_splice_init(from, &ret_folios);
>> + list_splice_init(&split_folios, from);
>> + no_split_folio_counting = true;
>> retry = 1;
>> - goto thp_subpage_migration;
>> + goto split_folio_migration;
>> }
>> - rc = nr_failed + nr_thp_failed;
>> + rc = nr_failed + nr_large_failed;
>> out:
>> /*
>> - * Put the permanent failure page back to migration list, they
>> + * Put the permanent failure folio back to migration list, they
>> * will be put back to the right list by the caller.
>> */
>> - list_splice(&ret_pages, from);
>> + list_splice(&ret_folios, from);
>> /*
>> - * Return 0 in case all subpages of fail-to-migrate THPs are
>> - * migrated successfully.
>> + * Return 0 in case all split folios of fail-to-migrate large folios
>> + * are migrated successfully.
>> */
>> if (list_empty(from))
>> rc = 0;
