alloc_contig_range() will fail if it finds a HugeTLB page within the range,
without a chance to handle them. Since HugeTLB pages can be migrated as any
LRU or Movable page, it does not make sense to bail out without trying.
Enable the interface to recognize in-use HugeTLB pages so we can migrate
them, and have much better chances to succeed the call.
Signed-off-by: Oscar Salvador <osalvador@xxxxxxx>
Reviewed-by: Mike Kravetz <mike.kravetz@xxxxxxxxxx>
---
include/linux/hugetlb.h | 5 +++--
mm/compaction.c | 12 +++++++++++-
mm/hugetlb.c | 22 +++++++++++++++++++---
mm/vmscan.c | 5 +++--
4 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index bcff86ca616f..a37b4ce86e58 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -583,7 +583,7 @@ struct huge_bootmem_page {
struct hstate *hstate;
};
-int isolate_or_dissolve_huge_page(struct page *page);
+int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list);
struct page *alloc_huge_page(struct vm_area_struct *vma,
unsigned long addr, int avoid_reserve);
struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
@@ -866,7 +866,8 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
#else /* CONFIG_HUGETLB_PAGE */
struct hstate {};
-static inline int isolate_or_dissolve_huge_page(struct page *page)
+static inline int isolate_or_dissolve_huge_page(struct page *page,
+ struct list_head *list)
{
return -ENOMEM;
}
diff --git a/mm/compaction.c b/mm/compaction.c
index 9f253fc3b4f9..6e47855fd154 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -910,7 +910,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
}
if (PageHuge(page) && cc->alloc_contig) {
- ret = isolate_or_dissolve_huge_page(page);
+ ret = isolate_or_dissolve_huge_page(page, &cc->migratepages);
/*
* Fail isolation in case isolate_or_dissolve_huge_page
@@ -927,6 +927,15 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
goto isolate_fail;
}
+ if (PageHuge(page)) {
+ /*
+ * Hugepage was successfully isolated and placed
+ * on the cc->migratepages list.
+ */
+ low_pfn += compound_nr(page) - 1;
+ goto isolate_success_no_list;
+ }
+
/*
* Ok, the hugepage was dissolved. Now these pages are
* Buddy and cannot be re-allocated because they are
@@ -1068,6 +1077,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
isolate_success:
list_add(&page->lru, &cc->migratepages);
+isolate_success_no_list:
cc->nr_migratepages += compound_nr(page);
nr_isolated += compound_nr(page);
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 3194c1bd9e32..11e86434d8bd 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2287,7 +2287,9 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page)
goto unlock;
} else if (page_count(old_page)) {
/*
- * Someone has grabbed the page, fail for now.
+ * Someone has grabbed the page, return -EBUSY so we give
+ * isolate_or_dissolve_huge_page a chance to handle an in-use
+ * page.
*/
ret = -EBUSY;
update_and_free_page(h, new_page);
@@ -2319,10 +2321,12 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page)
return ret;
}
-int isolate_or_dissolve_huge_page(struct page *page)
+int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
{
struct hstate *h;
struct page *head;
+ bool try_again = true;
+ int ret = -EBUSY;
/*
* The page might have been dissolved from under our feet, so make sure
@@ -2347,7 +2351,19 @@ int isolate_or_dissolve_huge_page(struct page *page)
if (hstate_is_gigantic(h))
return -ENOMEM;
- return alloc_and_dissolve_huge_page(h, head);
+retry:
+ if (page_count(head) && isolate_huge_page(head, list)) {
+ ret = 0;
+ } else if (!page_count(head)) {
+ ret = alloc_and_dissolve_huge_page(h, head);
+
+ if (ret == -EBUSY && try_again) {
+ try_again = false;
+ goto retry;
+ }
+ }
+
+ return ret;
}
struct page *alloc_huge_page(struct vm_area_struct *vma,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 562e87cbd7a1..42aaef30633e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1507,8 +1507,9 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
LIST_HEAD(clean_pages);
list_for_each_entry_safe(page, next, page_list, lru) {
- if (page_is_file_lru(page) && !PageDirty(page) &&
- !__PageMovable(page) && !PageUnevictable(page)) {
+ if (!PageHuge(page) && page_is_file_lru(page) &&
+ !PageDirty(page) && !__PageMovable(page) &&
+ !PageUnevictable(page)) {
ClearPageActive(page);
list_move(&page->lru, &clean_pages);
}
--
2.16.3
