Munlock isolates page from LRU and then looks for another mlock vma.
Thus we could could rechange page to second mlock without isolating.
This patch adds argument 'isolated' to mlock_vma_page() and passes this
flag through try_to_ummap as TTU_LRU_ISOLATED.
Signed-off-by: Konstantin Khlebnikov <khlebnikov@xxxxxxxxxxxxxx>
---
include/linux/rmap.h | 3 ++-
mm/gup.c | 2 +-
mm/huge_memory.c | 4 ++--
mm/internal.h | 6 ++++--
mm/ksm.c | 2 +-
mm/migrate.c | 2 +-
mm/mlock.c | 21 ++++++++++++---------
mm/rmap.c | 5 +++--
8 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 988d176472df..4552716ac3da 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -98,7 +98,8 @@ enum ttu_flags {
* do a final flush if necessary */
TTU_RMAP_LOCKED = 0x80, /* do not grab rmap lock:
* caller holds it */
- TTU_SPLIT_FREEZE = 0x100, /* freeze pte under splitting thp */
+ TTU_SPLIT_FREEZE = 0x100, /* freeze pte under splitting thp */
+ TTU_LRU_ISOLATED = 0x200, /* caller isolated page from LRU */
};
#ifdef CONFIG_MMU
diff --git a/mm/gup.c b/mm/gup.c
index f0accc229266..e0784e9022fe 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -306,7 +306,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
* know the page is still mapped, we don't even
* need to check for file-cache page truncation.
*/
- mlock_vma_page(vma, page);
+ mlock_vma_page(vma, page, false);
unlock_page(page);
}
}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 157faa231e26..7822997d765c 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1513,7 +1513,7 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
goto skip_mlock;
lru_add_drain();
if (page->mapping && !PageDoubleMap(page))
- mlock_vma_page(vma, page);
+ mlock_vma_page(vma, page, false);
unlock_page(page);
}
skip_mlock:
@@ -3009,7 +3009,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
page_add_file_rmap(new, true);
set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
- mlock_vma_page(vma, new);
+ mlock_vma_page(vma, new, false);
update_mmu_cache_pmd(vma, address, pvmw->pmd);
}
#endif
diff --git a/mm/internal.h b/mm/internal.h
index 9f91992ef281..1639fb581496 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -305,7 +305,8 @@ static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
/*
* must be called with vma's mmap_sem held for read or write, and page locked.
*/
-extern void mlock_vma_page(struct vm_area_struct *vma, struct page *page);
+extern void mlock_vma_page(struct vm_area_struct *vma, struct page *page,
+ bool isolated);
extern unsigned int munlock_vma_page(struct page *page);
/*
@@ -364,7 +365,8 @@ vma_address(struct page *page, struct vm_area_struct *vma)
#else /* !CONFIG_MMU */
static inline void clear_page_mlock(struct page *page) { }
-static inline void mlock_vma_page(struct vm_area_struct *, struct page *) { }
+static inline void mlock_vma_page(struct vm_area_struct *vma,
+ struct page *page, bool isolated) { }
static inline void mlock_migrate_page(struct page *new, struct page *old) { }
#endif /* !CONFIG_MMU */
diff --git a/mm/ksm.c b/mm/ksm.c
index cb5705d6f26c..bf2a748c5e64 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -1274,7 +1274,7 @@ static int try_to_merge_one_page(struct vm_area_struct *vma,
if (!PageMlocked(kpage)) {
unlock_page(page);
lock_page(kpage);
- mlock_vma_page(vma, kpage);
+ mlock_vma_page(vma, kpage, false);
page = kpage; /* for final unlock */
}
}
diff --git a/mm/migrate.c b/mm/migrate.c
index 1f6151cb7310..c13256ddc063 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -269,7 +269,7 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
page_add_file_rmap(new, false);
}
if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
- mlock_vma_page(vma, new);
+ mlock_vma_page(vma, new, false);
if (PageTransHuge(page) && PageMlocked(page))
clear_page_mlock(page);
diff --git a/mm/mlock.c b/mm/mlock.c
index 68f068711203..07a2ab4d6a6c 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -85,7 +85,8 @@ void clear_page_mlock(struct page *page)
* Mark page as mlocked if not already.
* If page on LRU, isolate and putback to move to unevictable list.
*/
-void mlock_vma_page(struct vm_area_struct *vma, struct page *page)
+void mlock_vma_page(struct vm_area_struct *vma, struct page *page,
+ bool isolated)
{
/* Serialize with page migration */
BUG_ON(!PageLocked(page));
@@ -97,15 +98,17 @@ void mlock_vma_page(struct vm_area_struct *vma, struct page *page)
mod_zone_page_state(page_zone(page), NR_MLOCK,
hpage_nr_pages(page));
count_vm_event(UNEVICTABLE_PGMLOCKED);
- if (!isolate_lru_page(page)) {
- /*
- * Force memory recharge to mlock user. Cannot
- * reclaim memory because called under pte lock.
- */
- mem_cgroup_try_recharge(page, vma->vm_mm,
- GFP_NOWAIT | __GFP_NOFAIL);
+
+ if (!isolated && isolate_lru_page(page))
+ return;
+ /*
+ * Force memory recharge to mlock user.
+ * Cannot reclaim memory because called under pte lock.
+ */
+ mem_cgroup_try_recharge(page, vma->vm_mm,
+ GFP_NOWAIT | __GFP_NOFAIL);
+ if (!isolated)
putback_lru_page(page);
- }
}
}
diff --git a/mm/rmap.c b/mm/rmap.c
index de88f4897c1d..0b21b27f3519 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1410,7 +1410,8 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
* Holding pte lock, we do *not* need
* mmap_sem here
*/
- mlock_vma_page(vma, page);
+ mlock_vma_page(vma, page,
+ !!(flags & TTU_LRU_ISOLATED));
}
ret = false;
page_vma_mapped_walk_done(&pvmw);
@@ -1752,7 +1753,7 @@ void try_to_munlock(struct page *page)
{
struct rmap_walk_control rwc = {
.rmap_one = try_to_unmap_one,
- .arg = (void *)TTU_MUNLOCK,
+ .arg = (void *)(TTU_MUNLOCK | TTU_LRU_ISOLATED),
.done = page_not_mapped,
.anon_lock = page_lock_anon_vma_read,
