We gather the pages that need to be unmapped in shrink_page_list. We batch
the unmap to reduce the frequency of acquisition of
the tree lock protecting the mapping's radix tree. This is
possible as successive pages likely share the same mapping in
__remove_mapping_batch routine. This avoids excessive cache bouncing of
the tree lock when page reclamations are occurring simultaneously.
Tim
---
Signed-off-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
---
diff --git a/mm/vmscan.c b/mm/vmscan.c
index aac5672..d4ab646 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -600,6 +600,85 @@ cannot_free:
return 0;
}
+/* Same as __remove_mapping, but batching operations to minimize locking */
+/* Pages to be unmapped should be locked first */
+static int __remove_mapping_batch(struct list_head *unmap_pages,
+ struct list_head *ret_pages,
+ struct list_head *free_pages)
+{
+ struct address_space *mapping, *next;
+ LIST_HEAD(swap_pages);
+ swp_entry_t swap;
+ struct page *page;
+ int nr_reclaimed;
+
+ mapping = NULL;
+ nr_reclaimed = 0;
+ while (!list_empty(unmap_pages)) {
+
+ page = lru_to_page(unmap_pages);
+ BUG_ON(!PageLocked(page));
+
+ list_del(&page->lru);
+ next = page_mapping(page);
+ if (mapping != next) {
+ if (mapping)
+ spin_unlock_irq(&mapping->tree_lock);
+ mapping = next;
+ spin_lock_irq(&mapping->tree_lock);
+ }
+
+ if (!page_freeze_refs(page, 2))
+ goto cannot_free;
+ if (unlikely(PageDirty(page))) {
+ page_unfreeze_refs(page, 2);
+ goto cannot_free;
+ }
+
+ if (PageSwapCache(page)) {
+ __delete_from_swap_cache(page);
+ /* swapcache_free need to be called without tree_lock */
+ list_add(&page->lru, &swap_pages);
+ } else {
+ void (*freepage)(struct page *);
+
+ freepage = mapping->a_ops->freepage;
+
+ __delete_from_page_cache(page);
+ mem_cgroup_uncharge_cache_page(page);
+
+ if (freepage != NULL)
+ freepage(page);
+
+ unlock_page(page);
+ nr_reclaimed++;
+ list_add(&page->lru, free_pages);
+ }
+ continue;
+cannot_free:
+ unlock_page(page);
+ list_add(&page->lru, ret_pages);
+ VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
+
+ }
+
+ if (mapping)
+ spin_unlock_irq(&mapping->tree_lock);
+
+ while (!list_empty(&swap_pages)) {
+ page = lru_to_page(&swap_pages);
+ list_del(&page->lru);
+
+ swap.val = page_private(page);
+ swapcache_free(swap, page);
+
+ unlock_page(page);
+ nr_reclaimed++;
+ list_add(&page->lru, free_pages);
+ }
+
+ return nr_reclaimed;
+}
/*
* Attempt to detach a locked page from its ->mapping. If it is dirty or if
* someone else has a ref on the page, abort and return 0. If it was
@@ -771,6 +850,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
{
LIST_HEAD(ret_pages);
LIST_HEAD(free_pages);
+ LIST_HEAD(unmap_pages);
int pgactivate = 0;
unsigned long nr_dirty = 0;
unsigned long nr_congested = 0;
@@ -969,17 +1049,13 @@ static unsigned long shrink_page_list(struct list_head *page_list,
}
}
- if (!mapping || !__remove_mapping(mapping, page))
+ if (!mapping)
goto keep_locked;
- /*
- * At this point, we have no other references and there is
- * no way to pick any more up (removed from LRU, removed
- * from pagecache). Can use non-atomic bitops now (and
- * we obviously don't have to worry about waking up a process
- * waiting on the page lock, because there are no references.
- */
- __clear_page_locked(page);
+ /* remove pages from mapping in batch at end of loop */
+ list_add(&page->lru, &unmap_pages);
+ continue;
+
free_it:
nr_reclaimed++;
@@ -1014,6 +1090,9 @@ keep_lumpy:
VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
}
+ nr_reclaimed += __remove_mapping_batch(&unmap_pages, &ret_pages,
+ &free_pages);
+
/*
* Tag a zone as congested if all the dirty pages encountered were
* backed by a congested BDI. In this case, reclaimers should just
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>