When the magazine is empty or full the zone lock is taken and a single
page is operated on. This makes the zone lock hotter than it needs to be
so batch allocations and frees from the zone. A larger number of pages
are taken when refilling the magazine to reduce the contention on the
zone->lock for IRQ-disabled callers. It's more likely that a workload will
notice contention on allocations than contentions on free although of
course this is workload dependant
Signed-off-by: Mel Gorman <mgorman@xxxxxxx>
---
mm/page_alloc.c | 172 +++++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 127 insertions(+), 45 deletions(-)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 9ed05a5..9426174 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -601,8 +601,6 @@ static inline void __free_one_page(struct page *page,
list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
out:
zone->free_area[order].nr_free++;
- if (unlikely(!is_migrate_isolate(migratetype)))
- __mod_zone_freepage_state(zone, 1 << order, migratetype);
}
static inline int free_pages_check(struct page *page)
@@ -634,6 +632,8 @@ static void free_one_page(struct zone *zone, struct page *page,
__count_vm_events(PGFREE, 1 << order);
__free_one_page(page, zone, order, migratetype);
+ if (unlikely(!is_migrate_isolate(migratetype)))
+ __mod_zone_freepage_state(zone, 1 << order, migratetype);
spin_unlock_irqrestore(&zone->lock, flags);
}
@@ -1093,6 +1093,87 @@ void mark_free_pages(struct zone *zone)
#endif /* CONFIG_PM */
#define MAGAZINE_LIMIT (1024)
+#define MAGAZINE_ALLOC_BATCH (384)
+#define MAGAZINE_FREE_BATCH (64)
+
+static
+struct page *__rmqueue_magazine(struct zone *zone, int migratetype)
+{
+ struct page *page;
+ struct free_area *area = &(zone->noirq_magazine);
+
+ if (list_empty(&area->free_list[migratetype]))
+ return NULL;
+
+ /* Page is available in the magazine, allocate it */
+ page = list_entry(area->free_list[migratetype].next, struct page, lru);
+ list_del(&page->lru);
+ area->nr_free--;
+ set_page_private(page, 0);
+
+ return page;
+}
+
+static void magazine_drain(struct zone *zone, int migratetype)
+{
+ struct free_area *area = &(zone->noirq_magazine);
+ struct list_head *list;
+ struct page *page;
+ unsigned int batch_free = 0;
+ unsigned int to_free = MAGAZINE_FREE_BATCH;
+ unsigned int nr_freed_cma = 0;
+ unsigned long flags;
+ LIST_HEAD(free_list);
+
+ if (area->nr_free < MAGAZINE_LIMIT) {
+ spin_unlock(&zone->magazine_lock);
+ return;
+ }
+
+ /* Free batch number of pages */
+ while (to_free) {
+ /*
+ * Removes pages from lists in a round-robin fashion. A
+ * batch_free count is maintained that is incremented when an
+ * empty list is encountered. This is so more pages are freed
+ * off fuller lists instead of spinning excessively around empty
+ * lists
+ */
+ do {
+ batch_free++;
+ if (++migratetype == MIGRATE_PCPTYPES)
+ migratetype = 0;
+ list = &area->free_list[migratetype];;
+ } while (list_empty(list));
+
+ /* This is the only non-empty list. Free them all. */
+ if (batch_free == MIGRATE_PCPTYPES)
+ batch_free = to_free;
+
+ do {
+ page = list_entry(list->prev, struct page, lru);
+ area->nr_free--;
+ set_page_private(page, 0);
+ list_move(&page->lru, &free_list);
+ if (is_migrate_isolate_page(zone, page))
+ nr_freed_cma++;
+ } while (--to_free && --batch_free && !list_empty(list));
+ }
+
+ /* Free the list of pages to the buddy allocator */
+ spin_unlock(&zone->magazine_lock);
+ spin_lock_irqsave(&zone->lock, flags);
+ while (!list_empty(&free_list)) {
+ page = list_entry(free_list.prev, struct page, lru);
+ list_del(&page->lru);
+ __free_one_page(page, zone, 0, get_freepage_migratetype(page));
+ }
+ __mod_zone_page_state(zone, NR_FREE_PAGES,
+ MAGAZINE_FREE_BATCH - nr_freed_cma);
+ if (nr_freed_cma)
+ __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_freed_cma);
+ spin_unlock_irqrestore(&zone->lock, flags);
+}
/*
* Free a 0-order page
@@ -1111,8 +1192,10 @@ void free_hot_cold_page(struct page *page, bool cold)
set_freepage_migratetype(page, migratetype);
/* magazine_lock is not safe against IRQs */
- if (in_interrupt() || irqs_disabled())
- goto free_one;
+ if (in_interrupt() || irqs_disabled()) {
+ free_one_page(zone, page, 0, migratetype);
+ return;
+ }
/* Put the free page on the magazine list */
spin_lock(&zone->magazine_lock);
@@ -1121,32 +1204,10 @@ void free_hot_cold_page(struct page *page, bool cold)
list_add(&page->lru, &area->free_list[migratetype]);
else
list_add_tail(&page->lru, &area->free_list[migratetype]);
- page = NULL;
-
- /* If the magazine is full, remove a cold page for the buddy list */
- if (area->nr_free > MAGAZINE_LIMIT) {
- struct list_head *list = &area->free_list[migratetype];
- int starttype = migratetype;
+ area->nr_free++;
- while (list_empty(list)) {
- if (++migratetype == MIGRATE_PCPTYPES)
- migratetype = 0;
- list = &area->free_list[migratetype];;
-
- WARN_ON_ONCE(starttype == migratetype);
- }
-
- page = list_entry(list->prev, struct page, lru);
- list_del(&page->lru);
- } else {
- area->nr_free++;
- }
- spin_unlock(&zone->magazine_lock);
-
-free_one:
- /* Free a page back to the buddy lists if necessary */
- if (page)
- free_one_page(zone, page, 0, migratetype);
+ /* Drain the magazine if necessary, releases the magazine lock */
+ magazine_drain(zone, migratetype);
}
/*
@@ -1261,25 +1322,46 @@ static
struct page *rmqueue_magazine(struct zone *zone, int migratetype)
{
struct page *page = NULL;
- struct free_area *area;
- /* Check if it is worth acquiring the lock */
- if (!zone->noirq_magazine.nr_free)
- return NULL;
-
- spin_lock(&zone->magazine_lock);
- area = &(zone->noirq_magazine);
- if (list_empty(&area->free_list[migratetype]))
- goto out;
+ /* Only acquire the lock if there is a reasonable chance of success */
+ if (zone->noirq_magazine.nr_free) {
+ spin_lock(&zone->magazine_lock);
+ page = __rmqueue_magazine(zone, migratetype);
+ spin_unlock(&zone->magazine_lock);
+ }
- /* Page is available in the magazine, allocate it */
- page = list_entry(area->free_list[migratetype].next, struct page, lru);
- list_del(&page->lru);
- area->nr_free--;
- set_page_private(page, 0);
+ /* Try refilling the magazine on allocaion failure */
+ if (!page) {
+ LIST_HEAD(alloc_list);
+ unsigned long flags;
+ struct free_area *area = &(zone->noirq_magazine);
+ unsigned int i;
+ unsigned int nr_alloced = 0;
+
+ spin_lock_irqsave(&zone->lock, flags);
+ for (i = 0; i < MAGAZINE_ALLOC_BATCH; i++) {
+ page = __rmqueue(zone, 0, migratetype);
+ if (!page)
+ break;
+ list_add_tail(&page->lru, &alloc_list);
+ nr_alloced++;
+ }
+ if (!is_migrate_cma(mt))
+ __mod_zone_page_state(zone, NR_FREE_PAGES, -nr_alloced);
+ else
+ __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -nr_alloced);
+ spin_unlock_irqrestore(&zone->lock, flags);
+
+ spin_lock(&zone->magazine_lock);
+ while (!list_empty(&alloc_list)) {
+ page = list_entry(alloc_list.next, struct page, lru);
+ list_move_tail(&page->lru, &area->free_list[migratetype]);
+ area->nr_free++;
+ }
+ page = __rmqueue_magazine(zone, migratetype);
+ spin_unlock(&zone->magazine_lock);
+ }
-out:
- spin_unlock(&zone->magazine_lock);
return page;
}
--
1.8.1.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>