This adds another case, as David points out, which is suitable for
mapping anonymous pte-mapped THPs by pmds. When all subpages are
PageAnonExclusive (PTEs may be either R/O or R/W), we can clear
PageAnonExclusive on all tail pages but the first (head) page and
collapse to a R/W PMD with VM_WRITE or a R/O PMD without VM_WRITE.
Signed-off-by: Xu Yu <xuyu@xxxxxxxxxxxxxxxxx>
---
mm/khugepaged.c | 23 ++++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 57e261387124..d843c1e3ec39 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1285,6 +1285,7 @@ static int collapse_pte_mapped_anon_thp(struct mm_struct *mm,
unsigned long addr;
int exclusive = 0;
bool writable = false;
+ rmap_t rmap_flags = RMAP_COMPOUND;
int result, i;
/* Fast check before locking folio if already PMD-mapped */
@@ -1366,8 +1367,14 @@ static int collapse_pte_mapped_anon_thp(struct mm_struct *mm,
* Case 1:
* No subpages are PageAnonExclusive (PTEs must be R/O), we can
* collapse into a R/O PMD without further action.
+ *
+ * Case 2:
+ * All subpages are PageAnonExclusive (PTEs may be either R/O or R/W),
+ * we clear PageAnonExclusive on all tail pages but the head page and
+ * collapse to a R/W PMD with VM_WRITE or a R/O PMD without VM_WRITE.
*/
- if (!(exclusive == 0 && !writable))
+ if (!((exclusive == 0 && !writable) ||
+ (exclusive == HPAGE_PMD_NR)))
goto up_write;
/* Collapse pmd entry */
@@ -1390,7 +1397,9 @@ static int collapse_pte_mapped_anon_thp(struct mm_struct *mm,
* false-negative page_mapped().
*/
folio_get(folio);
- page_add_anon_rmap(&folio->page, vma, haddr, RMAP_COMPOUND);
+ if (exclusive == HPAGE_PMD_NR)
+ rmap_flags |= RMAP_EXCLUSIVE;
+ page_add_anon_rmap(&folio->page, vma, haddr, rmap_flags);
start_pte = pte_offset_map_lock(mm, &pmdval, haddr, &ptl);
for (i = 0, addr = haddr, pte = start_pte;
@@ -1401,6 +1410,10 @@ static int collapse_pte_mapped_anon_thp(struct mm_struct *mm,
ptep_clear(mm, addr, pte);
subpage = vm_normal_page(vma, addr, pteval);
page_remove_rmap(subpage, vma, false);
+
+ /* Clear PageAnonExclusive on tail pages */
+ if (exclusive == HPAGE_PMD_NR && i != 0)
+ ClearPageAnonExclusive(subpage);
}
pte_unmap_unlock(start_pte, ptl);
folio_ref_sub(folio, HPAGE_PMD_NR);
@@ -1408,6 +1421,8 @@ static int collapse_pte_mapped_anon_thp(struct mm_struct *mm,
/* Install pmd entry */
pgtable = pmd_pgtable(pmdval);
pmdval = mk_huge_pmd(&folio->page, vma->vm_page_prot);
+ if (exclusive == HPAGE_PMD_NR)
+ pmdval = maybe_pmd_mkwrite(pmd_mkdirty(pmdval), vma);
spin_lock(pml);
pgtable_trans_huge_deposit(mm, pmd, pgtable);
set_pmd_at(mm, haddr, pmd, pmdval);
@@ -1595,7 +1610,9 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm,
out_unmap:
pte_unmap_unlock(pte, ptl);
- if (is_hpage && (exclusive == 0 && !writable)) {
+ if (is_hpage &&
+ ((exclusive == 0 && !writable) ||
+ (exclusive == HPAGE_PMD_NR))) {
int res;
res = collapse_pte_mapped_anon_thp(mm, vma, address,
--
2.37.1
