Similar to page_move_anon_rmap() except it can batch-move a range of
pages within a folio for increased efficiency. Will be used to enable
reusing multiple pages from a large anonymous folio in one go.
Signed-off-by: Ryan Roberts <ryan.roberts@xxxxxxx>
---
include/linux/rmap.h | 2 ++
mm/rmap.c | 40 ++++++++++++++++++++++++++++++----------
2 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 5c707f53d7b5..8cb0ba48d58f 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -190,6 +190,8 @@ typedef int __bitwise rmap_t;
* rmap interfaces called when adding or removing pte of page
*/
void page_move_anon_rmap(struct page *, struct vm_area_struct *);
+void folio_move_anon_rmap_range(struct folio *folio, struct page *page,
+ int nr, struct vm_area_struct *vma);
void page_add_anon_rmap(struct page *, struct vm_area_struct *,
unsigned long address, rmap_t flags);
void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
diff --git a/mm/rmap.c b/mm/rmap.c
index 5148a484f915..1cd8fb0b929f 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1103,19 +1103,22 @@ int folio_total_mapcount(struct folio *folio)
}
/**
- * page_move_anon_rmap - move a page to our anon_vma
- * @page: the page to move to our anon_vma
- * @vma: the vma the page belongs to
+ * folio_move_anon_rmap_range - batch-move a range of pages within a folio to
+ * our anon_vma; a more efficient version of page_move_anon_rmap().
+ * @folio: folio that owns the range of pages
+ * @page: the first page to move to our anon_vma
+ * @nr: number of pages to move to our anon_vma
+ * @vma: the vma the page belongs to
*
- * When a page belongs exclusively to one process after a COW event,
- * that page can be moved into the anon_vma that belongs to just that
- * process, so the rmap code will not search the parent or sibling
- * processes.
+ * When a range of pages belongs exclusively to one process after a COW event,
+ * those pages can be moved into the anon_vma that belongs to just that process,
+ * so the rmap code will not search the parent or sibling processes.
*/
-void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
+void folio_move_anon_rmap_range(struct folio *folio, struct page *page,
+ int nr, struct vm_area_struct *vma)
{
void *anon_vma = vma->anon_vma;
- struct folio *folio = page_folio(page);
+ int i;
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
VM_BUG_ON_VMA(!anon_vma, vma);
@@ -1127,7 +1130,24 @@ void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
* folio_test_anon()) will not see one without the other.
*/
WRITE_ONCE(folio->mapping, anon_vma);
- SetPageAnonExclusive(page);
+
+ for (i = 0; i < nr; i++)
+ SetPageAnonExclusive(page++);
+}
+
+/**
+ * page_move_anon_rmap - move a page to our anon_vma
+ * @page: the page to move to our anon_vma
+ * @vma: the vma the page belongs to
+ *
+ * When a page belongs exclusively to one process after a COW event,
+ * that page can be moved into the anon_vma that belongs to just that
+ * process, so the rmap code will not search the parent or sibling
+ * processes.
+ */
+void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
+{
+ folio_move_anon_rmap_range(page_folio(page), page, 1, vma);
}
/**
--
2.25.1
