After do_wp_page, the vma established a new mapping relationship with
the new folio, and released the mapping relationship with the old folio.
However, the avc binding relationship between the vma and the anon_vma
in old folio still exists, and old folio can still traverse the
unrelated vma during rmap. This is incorrent and will increase the
rmap redundant overhead.
This patch will remove the avc binding relationship between vma and
anon_vma in old folio, alleviating rmap overhead.
Signed-off-by: Zhiguo Jiang <justinjiang@xxxxxxxx>
---
include/linux/rmap.h | 1 +
mm/memory.c | 8 ++++++++
mm/rmap.c | 40 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 49 insertions(+)
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 91b5935e8485..8607d28a3146
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -257,6 +257,7 @@ void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages,
folio_remove_rmap_ptes(folio, page, 1, vma)
void folio_remove_rmap_pmd(struct folio *, struct page *,
struct vm_area_struct *);
+void folio_remove_anon_avc(struct folio *, struct vm_area_struct *);
void hugetlb_add_anon_rmap(struct folio *, struct vm_area_struct *,
unsigned long address, rmap_t flags);
diff --git a/mm/memory.c b/mm/memory.c
index 93c0c25433d0..4c89cb1cb73e
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3428,6 +3428,14 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
* old page will be flushed before it can be reused.
*/
folio_remove_rmap_pte(old_folio, vmf->page, vma);
+
+ /*
+ * If the new_folio's anon_vma is different from the
+ * old_folio's anon_vma, the avc binding relationship
+ * between vma and the old_folio's anon_vma is removed,
+ * avoiding rmap redundant overhead.
+ */
+ folio_remove_anon_avc(old_folio, vma);
}
/* Free the old page.. */
diff --git a/mm/rmap.c b/mm/rmap.c
index 1103a536e474..0b9111c59dc2
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1522,6 +1522,46 @@ void folio_add_file_rmap_pmd(struct folio *folio, struct page *page,
#endif
}
+void folio_remove_anon_avc(struct folio *folio,
+ struct vm_area_struct *vma)
+{
+ struct anon_vma *anon_vma = folio_anon_vma(folio);
+ pgoff_t pgoff_start, pgoff_end;
+ struct anon_vma_chain *avc;
+
+ /*
+ * Ensure that the vma's anon_vma and the folio's
+ * anon_vma exist and are not the same.
+ */
+ if (!folio_test_anon(folio) || unlikely(!anon_vma) ||
+ anon_vma == vma->anon_vma)
+ return;
+
+ pgoff_start = folio_pgoff(folio);
+ pgoff_end = pgoff_start + folio_nr_pages(folio) - 1;
+
+ if (!anon_vma_trylock_write(anon_vma))
+ return;
+
+ anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
+ pgoff_start, pgoff_end) {
+ /*
+ * Find the avc associated with vma from the folio's
+ * anon_vma and remove it.
+ */
+ if (avc->vma == vma) {
+ list_del(&avc->same_vma);
+ anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
+
+ if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root))
+ anon_vma->parent->num_children--;
+ anon_vma_chain_free(avc);
+ break;
+ }
+ }
+ anon_vma_unlock_write(anon_vma);
+}
+
static __always_inline void __folio_remove_rmap(struct folio *folio,
struct page *page, int nr_pages, struct vm_area_struct *vma,
enum rmap_level level)
--
2.39.0
