The patch titled
Subject: mm/gup: longterm pin migration cleanup
has been added to the -mm tree. Its filename is
mm-gup-longterm-pin-migration-cleanup.patch
This patch should soon appear at
https://ozlabs.org/~akpm/mmots/broken-out/mm-gup-longterm-pin-migration-cleanup.patch
and later at
https://ozlabs.org/~akpm/mmotm/broken-out/mm-gup-longterm-pin-migration-cleanup.patch
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next and is updated
there every 3-4 working days
------------------------------------------------------
From: Pavel Tatashin <pasha.tatashin@xxxxxxxxxx>
Subject: mm/gup: longterm pin migration cleanup
When pages are longterm pinned, we must migrated them out of movable zone.
The function that migrates them has a hidden loop with goto. The loop is
to retry on isolation failures, and after successful migration.
Make this code better by moving this loop to the caller.
Link: https://lkml.kernel.org/r/20210201153827.444374-13-pasha.tatashin@xxxxxxxxxx
Signed-off-by: Pavel Tatashin <pasha.tatashin@xxxxxxxxxx>
Reviewed-by: Jason Gunthorpe <jgg@xxxxxxxxxx>
Cc: Dan Williams <dan.j.williams@xxxxxxxxx>
Cc: David Hildenbrand <david@xxxxxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Ira Weiny <ira.weiny@xxxxxxxxx>
Cc: James Morris <jmorris@xxxxxxxxx>
Cc: Jason Gunthorpe <jgg@xxxxxxxx>
Cc: John Hubbard <jhubbard@xxxxxxxxxx>
Cc: Joonsoo Kim <iamjoonsoo.kim@xxxxxxx>
Cc: Matthew Wilcox <willy@xxxxxxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxx>
Cc: Mike Kravetz <mike.kravetz@xxxxxxxxxx>
Cc: Oscar Salvador <osalvador@xxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Sasha Levin <sashal@xxxxxxxxxx>
Cc: Steven Rostedt (VMware) <rostedt@xxxxxxxxxxx>
Cc: Tyler Hicks <tyhicks@xxxxxxxxxxxxxxxxxxx>
Cc: Vlastimil Babka <vbabka@xxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
mm/gup.c | 93 +++++++++++++++++++++--------------------------------
1 file changed, 37 insertions(+), 56 deletions(-)
--- a/mm/gup.c~mm-gup-longterm-pin-migration-cleanup
+++ a/mm/gup.c
@@ -1542,27 +1542,28 @@ struct page *get_dump_page(unsigned long
#endif /* CONFIG_ELF_CORE */
#ifdef CONFIG_MIGRATION
-static long check_and_migrate_movable_pages(struct mm_struct *mm,
- unsigned long start,
- unsigned long nr_pages,
+/*
+ * Check whether all pages are pinnable, if so return number of pages. If some
+ * pages are not pinnable, migrate them, and unpin all pages. Return zero if
+ * pages were migrated, or if some pages were not successfully isolated.
+ * Return negative error if migration fails.
+ */
+static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages,
- struct vm_area_struct **vmas,
unsigned int gup_flags)
{
- unsigned long i, isolation_error_count;
- bool drain_allow;
+ unsigned long i;
+ unsigned long isolation_error_count = 0;
+ bool drain_allow = true;
LIST_HEAD(movable_page_list);
- long ret = nr_pages;
- struct page *prev_head, *head;
+ long ret = 0;
+ struct page *prev_head = NULL;
+ struct page *head;
struct migration_target_control mtc = {
.nid = NUMA_NO_NODE,
.gfp_mask = GFP_USER | __GFP_NOWARN,
};
-check_again:
- prev_head = NULL;
- isolation_error_count = 0;
- drain_allow = true;
for (i = 0; i < nr_pages; i++) {
head = compound_head(pages[i]);
if (head == prev_head)
@@ -1600,47 +1601,27 @@ check_again:
* in the correct zone.
*/
if (list_empty(&movable_page_list) && !isolation_error_count)
- return ret;
+ return nr_pages;
+ if (gup_flags & FOLL_PIN) {
+ unpin_user_pages(pages, nr_pages);
+ } else {
+ for (i = 0; i < nr_pages; i++)
+ put_page(pages[i]);
+ }
if (!list_empty(&movable_page_list)) {
- /*
- * drop the above get_user_pages reference.
- */
- if (gup_flags & FOLL_PIN)
- unpin_user_pages(pages, nr_pages);
- else
- for (i = 0; i < nr_pages; i++)
- put_page(pages[i]);
-
ret = migrate_pages(&movable_page_list, alloc_migration_target,
NULL, (unsigned long)&mtc, MIGRATE_SYNC,
MR_LONGTERM_PIN);
- if (ret) {
- if (!list_empty(&movable_page_list))
- putback_movable_pages(&movable_page_list);
- return ret > 0 ? -ENOMEM : ret;
- }
-
- /* We unpinned pages before migration, pin them again */
- ret = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
- NULL, gup_flags);
- if (ret <= 0)
- return ret;
- nr_pages = ret;
+ if (ret && !list_empty(&movable_page_list))
+ putback_movable_pages(&movable_page_list);
}
- /*
- * check again because pages were unpinned, and we also might have
- * had isolation errors and need more pages to migrate.
- */
- goto check_again;
+ return ret > 0 ? -ENOMEM : ret;
}
#else
-static long check_and_migrate_movable_pages(struct mm_struct *mm,
- unsigned long start,
- unsigned long nr_pages,
+static long check_and_migrate_movable_pages(unsigned long nr_pages,
struct page **pages,
- struct vm_area_struct **vmas,
unsigned int gup_flags)
{
return nr_pages;
@@ -1658,22 +1639,22 @@ static long __gup_longterm_locked(struct
struct vm_area_struct **vmas,
unsigned int gup_flags)
{
- unsigned long flags = 0;
+ unsigned int flags;
long rc;
- if (gup_flags & FOLL_LONGTERM)
- flags = memalloc_pin_save();
-
- rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, NULL,
- gup_flags);
+ if (!(gup_flags & FOLL_LONGTERM))
+ return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
+ NULL, gup_flags);
+ flags = memalloc_pin_save();
+ do {
+ rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
+ NULL, gup_flags);
+ if (rc <= 0)
+ break;
+ rc = check_and_migrate_movable_pages(rc, pages, gup_flags);
+ } while (!rc);
+ memalloc_pin_restore(flags);
- if (gup_flags & FOLL_LONGTERM) {
- if (rc > 0)
- rc = check_and_migrate_movable_pages(mm, start, rc,
- pages, vmas,
- gup_flags);
- memalloc_pin_restore(flags);
- }
return rc;
}
_
Patches currently in -mm which might be from pasha.tatashin@xxxxxxxxxx are
mm-gup-dont-pin-migrated-cma-pages-in-movable-zone.patch
mm-gup-check-every-subpage-of-a-compound-page-during-isolation.patch
mm-gup-return-an-error-on-migration-failure.patch
mm-gup-check-for-isolation-errors.patch
mm-cma-rename-pf_memalloc_nocma-to-pf_memalloc_pin.patch
mm-apply-per-task-gfp-constraints-in-fast-path.patch
mm-honor-pf_memalloc_pin-for-all-movable-pages.patch
mm-gup-do-not-migrate-zero-page.patch
mm-gup-migrate-pinned-pages-out-of-movable-zone.patch
memory-hotplugrst-add-a-note-about-zone_movable-and-page-pinning.patch
mm-gup-change-index-type-to-long-as-it-counts-pages.patch
mm-gup-longterm-pin-migration-cleanup.patch
selftests-vm-gup_test-fix-test-flag.patch
selftests-vm-gup_test-test-faulting-in-kernel-and-verify-pinnable-pages.patch
