CMA added a is_migrate_isolate_page in the bulk page free path which
does a pageblock migratetype lookup for every page being drained. This
is only necessary when CMA is active so skip the expensive checks in the
normal case.
Signed-off-by: Mel Gorman <mgorman@xxxxxxx>
---
include/linux/mmzone.h | 8 ++++++--
include/linux/page-isolation.h | 7 ++++---
mm/page_alloc.c | 2 +-
mm/page_isolation.c | 27 +++++++++++++++++++++++----
4 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index e71e3a6..57f03b3 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -354,12 +354,16 @@ struct zone {
spinlock_t lock;
int all_unreclaimable; /* All pages pinned */
#if defined CONFIG_COMPACTION || defined CONFIG_CMA
+ unsigned long compact_cached_free_pfn;
+ unsigned long compact_cached_migrate_pfn;
+
/* Set to true when the PG_migrate_skip bits should be cleared */
bool compact_blockskip_flush;
/* pfns where compaction scanners should start */
- unsigned long compact_cached_free_pfn;
- unsigned long compact_cached_migrate_pfn;
+#endif
+#ifdef CONFIG_MEMORY_ISOLATION
+ bool memory_isolation_active;
#endif
#ifdef CONFIG_MEMORY_HOTPLUG
/* see spanned/present_pages for more description */
diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index 3fff8e7..81287bb 100644
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -2,16 +2,17 @@
#define __LINUX_PAGEISOLATION_H
#ifdef CONFIG_MEMORY_ISOLATION
-static inline bool is_migrate_isolate_page(struct page *page)
+static inline bool is_migrate_isolate_page(struct zone *zone, struct page *page)
{
- return get_pageblock_migratetype(page) == MIGRATE_ISOLATE;
+ return zone->memory_isolation_active &&
+ get_pageblock_migratetype(page) == MIGRATE_ISOLATE;
}
static inline bool is_migrate_isolate(int migratetype)
{
return migratetype == MIGRATE_ISOLATE;
}
#else
-static inline bool is_migrate_isolate_page(struct page *page)
+static inline bool is_migrate_isolate_page(struct zone *zone, struct page *page)
{
return false;
}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 4a07771..f170260 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -674,7 +674,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
/* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
__free_one_page(page, zone, 0, mt);
trace_mm_page_pcpu_drain(page, 0, mt);
- if (likely(!is_migrate_isolate_page(page))) {
+ if (likely(!is_migrate_isolate_page(zone, page))) {
__mod_zone_page_state(zone, NR_FREE_PAGES, 1);
if (is_migrate_cma(mt))
__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1);
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 383bdbb..9f0c068 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -118,6 +118,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
unsigned long pfn;
unsigned long undo_pfn;
struct page *page;
+ struct zone *zone = NULL;
+ unsigned long flags;
BUG_ON((start_pfn) & (pageblock_nr_pages - 1));
BUG_ON((end_pfn) & (pageblock_nr_pages - 1));
@@ -126,12 +128,20 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
pfn < end_pfn;
pfn += pageblock_nr_pages) {
page = __first_valid_page(pfn, pageblock_nr_pages);
- if (page &&
- set_migratetype_isolate(page, skip_hwpoisoned_pages)) {
- undo_pfn = pfn;
- goto undo;
+ if (page) {
+ if (!zone)
+ zone = page_zone(page);
+ if (set_migratetype_isolate(page,
+ skip_hwpoisoned_pages)) {
+ undo_pfn = pfn;
+ goto undo;
+ }
}
}
+
+ spin_lock_irqsave(&zone->lock, flags);
+ zone->memory_isolation_active = true;
+ spin_unlock_irqrestore(&zone->lock, flags);
return 0;
undo:
for (pfn = start_pfn;
@@ -150,6 +160,9 @@ int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
{
unsigned long pfn;
struct page *page;
+ struct zone *zone = NULL;
+ unsigned long flags;
+
BUG_ON((start_pfn) & (pageblock_nr_pages - 1));
BUG_ON((end_pfn) & (pageblock_nr_pages - 1));
for (pfn = start_pfn;
@@ -159,7 +172,13 @@ int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
if (!page || get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
continue;
unset_migratetype_isolate(page, migratetype);
+ if (!zone)
+ zone = page_zone(page);
}
+
+ spin_lock_irqsave(&zone->lock, flags);
+ zone->memory_isolation_active = true;
+ spin_unlock_irqrestore(&zone->lock, flags);
return 0;
}
/*
--
1.8.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>