With commit 09854ba94c6a ("mm: do_wp_page() simplification"), after
COW, the idle swap cache (neither the page nor the corresponding swap
entry is mapped by any process) will be left at the original position
in the LRU list. While it may be in the active list or the head of
the inactive list, so that vmscan may take more overhead or time to
reclaim these actually unused pages.
To help the page reclaiming, in this patch, after COW, the idle swap
cache will be tried to be moved to the tail of the inactive LRU list.
To avoid to introduce much overhead to the hot COW code path, all
locks are acquired with try locking.
To test the patch, we used pmbench memory accessing benchmark with
working-set larger than available memory on a 2-socket Intel server
with a NVMe SSD as swap device. Test results shows that the pmbench
score increases up to 21.8% with the decreased size of swap cache and
swapin throughput.
Signed-off-by: "Huang, Ying" <ying.huang@xxxxxxxxx>
Suggested-by: Matthew Wilcox <willy@xxxxxxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: Peter Xu <peterx@xxxxxxxxxx>
Cc: Hugh Dickins <hughd@xxxxxxxxxx>
Cc: Johannes Weiner <hannes@xxxxxxxxxxx>
Cc: Mel Gorman <mgorman@xxxxxxx>
Cc: Rik van Riel <riel@xxxxxxxxxxx>
Cc: Andrea Arcangeli <aarcange@xxxxxxxxxx>
Cc: Michal Hocko <mhocko@xxxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxx>
Cc: Tim Chen <tim.c.chen@xxxxxxxxx>
V2:
- Move trylock_page() to try_to_free_idle_swapcache() per Rik and
Linus' comments.
- Fix PageLRU() checking.
- Fix THP processing.
- Rename the function.
---
include/linux/memcontrol.h | 10 ++++++++++
include/linux/swap.h | 3 +++
mm/memcontrol.c | 12 ++++++++++++
mm/memory.c | 2 ++
mm/swapfile.c | 39 ++++++++++++++++++++++++++++++++++++++
5 files changed, 66 insertions(+)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 3cc18c2176e7..d6c6ff69b586 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -760,6 +760,7 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
struct lruvec *lock_page_lruvec(struct page *page);
struct lruvec *lock_page_lruvec_irq(struct page *page);
+struct lruvec *trylock_page_lruvec_irq(struct page *page);
struct lruvec *lock_page_lruvec_irqsave(struct page *page,
unsigned long *flags);
@@ -1250,6 +1251,15 @@ static inline struct lruvec *lock_page_lruvec_irq(struct page *page)
return &pgdat->__lruvec;
}
+static inline struct lruvec *trylock_page_lruvec_irq(struct page *page)
+{
+ struct pglist_data *pgdat = page_pgdat(page);
+
+ if (spin_trylock_irq(&pgdat->__lruvec.lru_lock))
+ return &pgdat->__lruvec;
+ return NULL;
+}
+
static inline struct lruvec *lock_page_lruvec_irqsave(struct page *page,
unsigned long *flagsp)
{
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 032485ee7597..dbef99233736 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -509,6 +509,7 @@ extern struct swap_info_struct *page_swap_info(struct page *);
extern struct swap_info_struct *swp_swap_info(swp_entry_t entry);
extern bool reuse_swap_page(struct page *, int *);
extern int try_to_free_swap(struct page *);
+extern void deactivate_idle_swapcache(struct page *page);
struct backing_dev_info;
extern int init_swap_address_space(unsigned int type, unsigned long nr_pages);
extern void exit_swap_address_space(unsigned int type);
@@ -677,6 +678,8 @@ static inline int try_to_free_swap(struct page *page)
return 0;
}
+static inline void deactivate_idle_swapcache(struct page *page) {}
+
static inline swp_entry_t get_swap_page(struct page *page)
{
swp_entry_t entry;
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index cb864f87b01d..a192476fc55c 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1213,6 +1213,18 @@ struct lruvec *lock_page_lruvec_irq(struct page *page)
return lruvec;
}
+struct lruvec *trylock_page_lruvec_irq(struct page *page)
+{
+ struct lruvec *lruvec;
+
+ lruvec = mem_cgroup_page_lruvec(page);
+ if (spin_trylock_irq(&lruvec->lru_lock)) {
+ lruvec_memcg_debug(lruvec, page);
+ return lruvec;
+ }
+ return NULL;
+}
+
struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
{
struct lruvec *lruvec;
diff --git a/mm/memory.c b/mm/memory.c
index 2b7ffcbca175..8a4b1ccd6879 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3104,6 +3104,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
munlock_vma_page(old_page);
unlock_page(old_page);
}
+ if (page_copied && PageSwapCache(old_page) && !page_mapped(old_page))
+ deactivate_idle_swapcache(old_page);
put_page(old_page);
}
return page_copied ? VM_FAULT_WRITE : 0;
diff --git a/mm/swapfile.c b/mm/swapfile.c
index cbb4c0795284..0b390b14c5ae 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -40,6 +40,7 @@
#include <linux/swap_slots.h>
#include <linux/sort.h>
#include <linux/completion.h>
+#include <linux/mm_inline.h>
#include <asm/tlbflush.h>
#include <linux/swapops.h>
@@ -1748,6 +1749,44 @@ int try_to_free_swap(struct page *page)
return 1;
}
+static inline void locked_deactivate_idle_swapcache(struct page *page)
+{
+ struct lruvec *lruvec;
+
+ page = compound_head(page);
+ if (!PageSwapCache(page))
+ return;
+ if (PageWriteback(page))
+ return;
+ if (page_mapped(page))
+ return;
+ if (!PageLRU(page))
+ return;
+ if (page_swapped(page))
+ return;
+ lruvec = trylock_page_lruvec_irq(page);
+ if (!lruvec)
+ return;
+
+ if (TestClearPageLRU(page)) {
+ del_page_from_lru_list(page, lruvec);
+ ClearPageActive(page);
+ ClearPageReferenced(page);
+ add_page_to_lru_list_tail(page, lruvec);
+ SetPageLRU(page);
+ }
+
+ unlock_page_lruvec_irq(lruvec);
+}
+
+void deactivate_idle_swapcache(struct page *page)
+{
+ if (trylock_page(page)) {
+ locked_deactivate_idle_swapcache(page);
+ unlock_page(page);
+ }
+}
+
/*
* Free the swap entry like above, but also try to
* free the page cache entry if it is the last user.
--
2.30.2
