A new mechanism, LUF(Lazy Unmap Flush), defers tlb flush until folios
that have been unmapped and freed, eventually get allocated again. It's
safe for folios that had been mapped read only and were unmapped, since
the contents of the folios don't change while staying in pcp or buddy
so we can still read the data through the stale tlb entries.
Applied the mechanism to unmapping during folio reclaim.
Signed-off-by: Byungchul Park <byungchul@xxxxxx>
---
include/linux/rmap.h | 5 +++--
mm/rmap.c | 5 ++++-
mm/vmscan.c | 21 ++++++++++++++++++++-
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 1898a2c1c087..9ca752f8de97 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -658,7 +658,7 @@ int folio_referenced(struct folio *, int is_locked,
struct mem_cgroup *memcg, unsigned long *vm_flags);
bool try_to_migrate(struct folio *folio, enum ttu_flags flags);
-void try_to_unmap(struct folio *, enum ttu_flags flags);
+bool try_to_unmap(struct folio *, enum ttu_flags flags);
int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
unsigned long end, struct page **pages,
@@ -777,8 +777,9 @@ static inline int folio_referenced(struct folio *folio, int is_locked,
return 0;
}
-static inline void try_to_unmap(struct folio *folio, enum ttu_flags flags)
+static inline bool try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
+ return false;
}
static inline int folio_mkclean(struct folio *folio)
diff --git a/mm/rmap.c b/mm/rmap.c
index d25ae20a47b5..571e337af448 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -2237,10 +2237,11 @@ static int folio_not_mapped(struct folio *folio)
* Tries to remove all the page table entries which are mapping this
* folio. It is the caller's responsibility to check if the folio is
* still mapped if needed (use TTU_SYNC to prevent accounting races).
+ * Return true if all the mappings are read-only, otherwise false.
*
* Context: Caller must hold the folio lock.
*/
-void try_to_unmap(struct folio *folio, enum ttu_flags flags)
+bool try_to_unmap(struct folio *folio, enum ttu_flags flags)
{
struct rmap_walk_control rwc = {
.rmap_one = try_to_unmap_one,
@@ -2265,6 +2266,8 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags)
fold_ubc(tlb_ubc_luf, tlb_ubc_ro);
else
fold_ubc(tlb_ubc, tlb_ubc_ro);
+
+ return can_luf;
}
/*
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bb0ff11f9ec9..4e2e9d07cd96 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1031,14 +1031,17 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
struct reclaim_stat *stat, bool ignore_references)
{
struct folio_batch free_folios;
+ struct folio_batch free_folios_luf;
LIST_HEAD(ret_folios);
LIST_HEAD(demote_folios);
unsigned int nr_reclaimed = 0;
unsigned int pgactivate = 0;
bool do_demote_pass;
struct swap_iocb *plug = NULL;
+ unsigned short int ugen;
folio_batch_init(&free_folios);
+ folio_batch_init(&free_folios_luf);
memset(stat, 0, sizeof(*stat));
cond_resched();
do_demote_pass = can_demote(pgdat->node_id, sc);
@@ -1050,6 +1053,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
enum folio_references references = FOLIOREF_RECLAIM;
bool dirty, writeback;
unsigned int nr_pages;
+ bool can_luf = false;
cond_resched();
@@ -1292,7 +1296,7 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
if (folio_test_large(folio) && list_empty(&folio->_deferred_list))
flags |= TTU_SYNC;
- try_to_unmap(folio, flags);
+ can_luf = try_to_unmap(folio, flags);
if (folio_mapped(folio)) {
stat->nr_unmap_fail += nr_pages;
if (!was_swapbacked &&
@@ -1457,6 +1461,18 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
if (folio_test_large(folio) &&
folio_test_large_rmappable(folio))
folio_undo_large_rmappable(folio);
+
+ if (can_luf) {
+ if (folio_batch_add(&free_folios_luf, folio) == 0) {
+ mem_cgroup_uncharge_folios(&free_folios_luf);
+ ugen = try_to_unmap_luf();
+ if (!ugen)
+ try_to_unmap_flush();
+ free_unref_folios(&free_folios_luf, ugen);
+ }
+ continue;
+ }
+
if (folio_batch_add(&free_folios, folio) == 0) {
mem_cgroup_uncharge_folios(&free_folios);
try_to_unmap_flush();
@@ -1526,8 +1542,11 @@ static unsigned int shrink_folio_list(struct list_head *folio_list,
pgactivate = stat->nr_activate[0] + stat->nr_activate[1];
mem_cgroup_uncharge_folios(&free_folios);
+ mem_cgroup_uncharge_folios(&free_folios_luf);
+ ugen = try_to_unmap_luf();
try_to_unmap_flush();
free_unref_folios(&free_folios, 0);
+ free_unref_folios(&free_folios_luf, ugen);
list_splice(&ret_folios, folio_list);
count_vm_events(PGACTIVATE, pgactivate);
--
2.17.1
