From: Alexander Duyck <alexander.h.duyck@xxxxxxxxxxxxxxx>
Add a pointer we shall call "membrane" which represents the upper boundary
between the "raw" and "treated" pages. The general idea is that in order
for a page to cross from one side of the membrane to the other it will need
to go through the aeration treatment.
By doing this we should be able to make certain that we keep the treated
pages as one contiguous block within each free list. While treating the
pages there may be two, but the two should merge into one before we
complete the migratetype and allow it to fall back into the "settling"
state.
Signed-off-by: Alexander Duyck <alexander.h.duyck@xxxxxxxxxxxxxxx>
---
include/linux/mmzone.h | 38 ++++++++++++++++++++++++++++++++++++++
mm/page_alloc.c | 14 ++++++++++++--
2 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a55fe6d2f63c..be996e8ca6b5 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -87,10 +87,28 @@ static inline bool is_migrate_movable(int mt)
get_pfnblock_flags_mask(page, page_to_pfn(page), \
PB_migrate_end, MIGRATETYPE_MASK)
+/*
+ * The treatment state indicates the current state of the region pointed to
+ * by the treatment_mt and the membrane pointer. The general idea is that
+ * when we are in the "SETTLING" state the treatment area is contiguous and
+ * it is safe to move on to treating another migratetype. If we are in the
+ * "AERATING" state then the region is being actively processed and we
+ * would cause issues such as potentially isolating a section of raw pages
+ * between two sections of treated pages if we were to move onto another
+ * migratetype.
+ */
+enum treatment_state {
+ TREATMENT_SETTLING,
+ TREATMENT_AERATING,
+};
+
struct free_area {
struct list_head free_list[MIGRATE_TYPES];
unsigned long nr_free_raw;
unsigned long nr_free_treated;
+ struct list_head *membrane;
+ u8 treatment_mt;
+ u8 treatment_state;
};
/* Used for pages not on another list */
@@ -113,6 +131,19 @@ static inline void add_to_free_area_tail(struct page *page, struct free_area *ar
list_add_tail(&page->lru, &area->free_list[migratetype]);
}
+static inline void
+add_to_free_area_treated(struct page *page, struct free_area *area,
+ int migratetype)
+{
+ area->nr_free_treated++;
+
+ BUG_ON(area->treatment_mt != migratetype);
+
+ /* Insert page above membrane, then move membrane to the page */
+ list_add_tail(&page->lru, area->membrane);
+ area->membrane = &page->lru;
+}
+
/* Used for pages which are on another list */
static inline void move_to_free_area(struct page *page, struct free_area *area,
int migratetype)
@@ -135,6 +166,10 @@ static inline void move_to_free_area(struct page *page, struct free_area *area,
area->nr_free_raw++;
}
+ /* push membrane back if we removed the upper boundary */
+ if (area->membrane == &page->lru)
+ area->membrane = page->lru.next;
+
list_move(&page->lru, &area->free_list[migratetype]);
}
@@ -153,6 +188,9 @@ static inline void del_page_from_free_area(struct page *page,
else
area->nr_free_raw--;
+ if (area->membrane == &page->lru)
+ area->membrane = page->lru.next;
+
list_del(&page->lru);
__ClearPageBuddy(page);
__ResetPageTreated(page);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index f6c067c6c784..f4a629b6af96 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -989,6 +989,11 @@ static inline void __free_one_page(struct page *page,
set_page_order(page, order);
area = &zone->free_area[order];
+ if (PageTreated(page)) {
+ add_to_free_area_treated(page, area, migratetype);
+ return;
+ }
+
if (buddy_merge_likely(pfn, buddy_pfn, page, order) ||
is_shuffle_tail_page(order))
add_to_free_area_tail(page, area, migratetype);
@@ -5961,8 +5966,13 @@ static void __meminit zone_init_free_lists(struct zone *zone)
INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
for (order = MAX_ORDER; order--; ) {
- zone->free_area[order].nr_free_raw = 0;
- zone->free_area[order].nr_free_treated = 0;
+ struct free_area *area = &zone->free_area[order];
+
+ area->nr_free_raw = 0;
+ area->nr_free_treated = 0;
+ area->treatment_mt = 0;
+ area->treatment_state = TREATMENT_SETTLING;
+ area->membrane = &area->free_list[0];
}
}
