The patch titled
Subject: mm: pgtable: introduce pagetable_dtor()
has been added to the -mm mm-unstable branch. Its filename is
mm-pgtable-introduce-pagetable_dtor.patch
This patch will shortly appear at
https://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new.git/tree/patches/mm-pgtable-introduce-pagetable_dtor.patch
This patch will later appear in the mm-unstable branch at
git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Before you just go and hit "reply", please:
a) Consider who else should be cc'ed
b) Prefer to cc a suitable mailing list as well
c) Ideally: find the original patch on the mailing list and do a
reply-to-all to that, adding suitable additional cc's
*** Remember to use Documentation/process/submit-checklist.rst when testing your code ***
The -mm tree is included into linux-next via the mm-everything
branch at git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
and is updated there every 2-3 working days
------------------------------------------------------
From: Qi Zheng <zhengqi.arch@xxxxxxxxxxxxx>
Subject: mm: pgtable: introduce pagetable_dtor()
Date: Mon, 23 Dec 2024 17:40:53 +0800
The pagetable_p*_dtor() are exactly the same except for the handling of
ptlock. If we make ptlock_free() handle the case where ptdesc->ptl is
NULL and remove VM_BUG_ON_PAGE() from pmd_ptlock_free(), we can unify
pagetable_p*_dtor() into one function. Let's introduce pagetable_dtor()
to do this.
Later, pagetable_dtor() will be moved to tlb_remove_ptdesc(), so that
ptlock and page table pages can be freed together (regardless of whether
RCU is used). This prevents the use-after-free problem where the ptlock
is freed immediately but the page table pages is freed later via RCU.
Link: https://lkml.kernel.org/r/8ada95453180c71b7fca92b9a9f11fa0f92d45a6.1734945104.git.zhengqi.arch@xxxxxxxxxxxxx
Signed-off-by: Qi Zheng <zhengqi.arch@xxxxxxxxxxxxx>
Originally-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Cc: Alexander Gordeev <agordeev@xxxxxxxxxxxxx>
Cc: Aneesh Kumar K.V (Arm) <aneesh.kumar@xxxxxxxxxx>
Cc: Arnd Bergmann <arnd@xxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Cc: David Hildenbrand <david@xxxxxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Hugh Dickins <hughd@xxxxxxxxxx>
Cc: Jann Horn <jannh@xxxxxxxxxx>
Cc: Kevin Brodsky <kevin.brodsky@xxxxxxx>
Cc: Lorenzo Stoakes <lorenzo.stoakes@xxxxxxxxxx>
Cc: Matthew Wilcox (Oracle) <willy@xxxxxxxxxxxxx>
Cc: Mike Rapoport (Microsoft) <rppt@xxxxxxxxxx>
Cc: Muchun Song <muchun.song@xxxxxxxxx>
Cc: Nicholas Piggin <npiggin@xxxxxxxxx>
Cc: Ryan Roberts <ryan.roberts@xxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Vishal Moola (Oracle) <vishal.moola@xxxxxxxxx>
Cc: Will Deacon <will@xxxxxxxxxx>
Cc: Yu Zhao <yuzhao@xxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
Documentation/mm/split_page_table_lock.rst | 4 -
arch/arm/include/asm/tlb.h | 4 -
arch/arm64/include/asm/tlb.h | 8 +-
arch/csky/include/asm/pgalloc.h | 2
arch/hexagon/include/asm/pgalloc.h | 2
arch/loongarch/include/asm/pgalloc.h | 2
arch/m68k/include/asm/mcf_pgalloc.h | 4 -
arch/m68k/include/asm/sun3_pgalloc.h | 2
arch/m68k/mm/motorola.c | 2
arch/mips/include/asm/pgalloc.h | 2
arch/nios2/include/asm/pgalloc.h | 2
arch/openrisc/include/asm/pgalloc.h | 2
arch/powerpc/mm/book3s64/mmu_context.c | 2
arch/powerpc/mm/book3s64/pgtable.c | 2
arch/powerpc/mm/pgtable-frag.c | 4 -
arch/riscv/include/asm/pgalloc.h | 8 +-
arch/riscv/mm/init.c | 4 -
arch/s390/include/asm/pgalloc.h | 6 +-
arch/s390/include/asm/tlb.h | 6 +-
arch/s390/mm/pgalloc.c | 2
arch/sh/include/asm/pgalloc.h | 2
arch/sparc/mm/init_64.c | 2
arch/sparc/mm/srmmu.c | 2
arch/um/include/asm/pgalloc.h | 6 +-
arch/x86/mm/pgtable.c | 12 ++--
include/asm-generic/pgalloc.h | 8 +-
include/linux/mm.h | 52 +++----------------
mm/memory.c | 3 -
28 files changed, 62 insertions(+), 95 deletions(-)
--- a/arch/arm64/include/asm/tlb.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/arm64/include/asm/tlb.h
@@ -82,7 +82,7 @@ static inline void __pte_free_tlb(struct
{
struct ptdesc *ptdesc = page_ptdesc(pte);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
@@ -92,7 +92,7 @@ static inline void __pmd_free_tlb(struct
{
struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
#endif
@@ -106,7 +106,7 @@ static inline void __pud_free_tlb(struct
if (!pgtable_l4_enabled())
return;
- pagetable_pud_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
#endif
@@ -120,7 +120,7 @@ static inline void __p4d_free_tlb(struct
if (!pgtable_l5_enabled())
return;
- pagetable_p4d_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
}
#endif
--- a/arch/arm/include/asm/tlb.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/arm/include/asm/tlb.h
@@ -41,7 +41,7 @@ __pte_free_tlb(struct mmu_gather *tlb, p
{
struct ptdesc *ptdesc = page_ptdesc(pte);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
#ifndef CONFIG_ARM_LPAE
/*
@@ -61,7 +61,7 @@ __pmd_free_tlb(struct mmu_gather *tlb, p
#ifdef CONFIG_ARM_LPAE
struct ptdesc *ptdesc = virt_to_ptdesc(pmdp);
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
tlb_remove_ptdesc(tlb, ptdesc);
#endif
}
--- a/arch/csky/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/csky/include/asm/pgalloc.h
@@ -63,7 +63,7 @@ static inline pgd_t *pgd_alloc(struct mm
#define __pte_free_tlb(tlb, pte, address) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc(tlb, page_ptdesc(pte)); \
} while (0)
--- a/arch/hexagon/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/hexagon/include/asm/pgalloc.h
@@ -89,7 +89,7 @@ static inline void pmd_populate_kernel(s
#define __pte_free_tlb(tlb, pte, addr) \
do { \
- pagetable_pte_dtor((page_ptdesc(pte))); \
+ pagetable_dtor((page_ptdesc(pte))); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)
--- a/arch/loongarch/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/loongarch/include/asm/pgalloc.h
@@ -57,7 +57,7 @@ static inline pte_t *pte_alloc_one_kerne
#define __pte_free_tlb(tlb, pte, address) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
} while (0)
--- a/arch/m68k/include/asm/mcf_pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/m68k/include/asm/mcf_pgalloc.h
@@ -37,7 +37,7 @@ static inline void __pte_free_tlb(struct
{
struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
@@ -61,7 +61,7 @@ static inline void pte_free(struct mm_st
{
struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
--- a/arch/m68k/include/asm/sun3_pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/m68k/include/asm/sun3_pgalloc.h
@@ -19,7 +19,7 @@ extern const char bad_pmd_string[];
#define __pte_free_tlb(tlb, pte, addr) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
} while (0)
--- a/arch/m68k/mm/motorola.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/m68k/mm/motorola.c
@@ -201,7 +201,7 @@ int free_pointer_table(void *table, int
list_del(dp);
mmu_page_dtor((void *)page);
if (type == TABLE_PTE)
- pagetable_pte_dtor(virt_to_ptdesc((void *)page));
+ pagetable_dtor(virt_to_ptdesc((void *)page));
free_page (page);
return 1;
} else if (ptable_list[type].next != dp) {
--- a/arch/mips/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/mips/include/asm/pgalloc.h
@@ -56,7 +56,7 @@ static inline void pgd_free(struct mm_st
#define __pte_free_tlb(tlb, pte, address) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), page_ptdesc(pte)); \
} while (0)
--- a/arch/nios2/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/nios2/include/asm/pgalloc.h
@@ -30,7 +30,7 @@ extern pgd_t *pgd_alloc(struct mm_struct
#define __pte_free_tlb(tlb, pte, addr) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)
--- a/arch/openrisc/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/openrisc/include/asm/pgalloc.h
@@ -68,7 +68,7 @@ extern pte_t *pte_alloc_one_kernel(struc
#define __pte_free_tlb(tlb, pte, addr) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)
--- a/arch/powerpc/mm/book3s64/mmu_context.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/powerpc/mm/book3s64/mmu_context.c
@@ -253,7 +253,7 @@ static void pmd_frag_destroy(void *pmd_f
count = ((unsigned long)pmd_frag & ~PAGE_MASK) >> PMD_FRAG_SIZE_SHIFT;
/* We allow PTE_FRAG_NR fragments from a PTE page */
if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
}
--- a/arch/powerpc/mm/book3s64/pgtable.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/powerpc/mm/book3s64/pgtable.c
@@ -477,7 +477,7 @@ void pmd_fragment_free(unsigned long *pm
BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0);
if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) {
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
}
--- a/arch/powerpc/mm/pgtable-frag.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/powerpc/mm/pgtable-frag.c
@@ -25,7 +25,7 @@ void pte_frag_destroy(void *pte_frag)
count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
/* We allow PTE_FRAG_NR fragments from a PTE page */
if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) {
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
}
@@ -111,7 +111,7 @@ static void pte_free_now(struct rcu_head
struct ptdesc *ptdesc;
ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
--- a/arch/riscv/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/riscv/include/asm/pgalloc.h
@@ -100,7 +100,7 @@ static inline void __pud_free_tlb(struct
if (pgtable_l4_enabled) {
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
- pagetable_pud_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, ptdesc);
}
}
@@ -111,7 +111,7 @@ static inline void __p4d_free_tlb(struct
if (pgtable_l5_enabled) {
struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
- pagetable_p4d_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d));
}
}
@@ -144,7 +144,7 @@ static inline void __pmd_free_tlb(struct
{
struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, ptdesc);
}
@@ -155,7 +155,7 @@ static inline void __pte_free_tlb(struct
{
struct ptdesc *ptdesc = page_ptdesc(pte);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
riscv_tlb_remove_ptdesc(tlb, ptdesc);
}
#endif /* CONFIG_MMU */
--- a/arch/riscv/mm/init.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/riscv/mm/init.c
@@ -1558,7 +1558,7 @@ static void __meminit free_pte_table(pte
return;
}
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
if (PageReserved(page))
free_reserved_page(page);
else
@@ -1580,7 +1580,7 @@ static void __meminit free_pmd_table(pmd
}
if (!is_vmemmap)
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
if (PageReserved(page))
free_reserved_page(page);
else
--- a/arch/s390/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/s390/include/asm/pgalloc.h
@@ -66,7 +66,7 @@ static inline void p4d_free(struct mm_st
if (mm_p4d_folded(mm))
return;
- pagetable_p4d_dtor(virt_to_ptdesc(p4d));
+ pagetable_dtor(virt_to_ptdesc(p4d));
crst_table_free(mm, (unsigned long *) p4d);
}
@@ -87,7 +87,7 @@ static inline void pud_free(struct mm_st
if (mm_pud_folded(mm))
return;
- pagetable_pud_dtor(virt_to_ptdesc(pud));
+ pagetable_dtor(virt_to_ptdesc(pud));
crst_table_free(mm, (unsigned long *) pud);
}
@@ -109,7 +109,7 @@ static inline void pmd_free(struct mm_st
{
if (mm_pmd_folded(mm))
return;
- pagetable_pmd_dtor(virt_to_ptdesc(pmd));
+ pagetable_dtor(virt_to_ptdesc(pmd));
crst_table_free(mm, (unsigned long *) pmd);
}
--- a/arch/s390/include/asm/tlb.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/s390/include/asm/tlb.h
@@ -102,7 +102,7 @@ static inline void pmd_free_tlb(struct m
{
if (mm_pmd_folded(tlb->mm))
return;
- pagetable_pmd_dtor(virt_to_ptdesc(pmd));
+ pagetable_dtor(virt_to_ptdesc(pmd));
__tlb_adjust_range(tlb, address, PAGE_SIZE);
tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1;
@@ -122,7 +122,7 @@ static inline void pud_free_tlb(struct m
{
if (mm_pud_folded(tlb->mm))
return;
- pagetable_pud_dtor(virt_to_ptdesc(pud));
+ pagetable_dtor(virt_to_ptdesc(pud));
tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1;
tlb->cleared_p4ds = 1;
@@ -141,7 +141,7 @@ static inline void p4d_free_tlb(struct m
{
if (mm_p4d_folded(tlb->mm))
return;
- pagetable_p4d_dtor(virt_to_ptdesc(p4d));
+ pagetable_dtor(virt_to_ptdesc(p4d));
__tlb_adjust_range(tlb, address, PAGE_SIZE);
tlb->mm->context.flush_mm = 1;
tlb->freed_tables = 1;
--- a/arch/s390/mm/pgalloc.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/s390/mm/pgalloc.c
@@ -182,7 +182,7 @@ unsigned long *page_table_alloc(struct m
static void pagetable_pte_dtor_free(struct ptdesc *ptdesc)
{
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
--- a/arch/sh/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/sh/include/asm/pgalloc.h
@@ -34,7 +34,7 @@ static inline void pmd_populate(struct m
#define __pte_free_tlb(tlb, pte, addr) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)
--- a/arch/sparc/mm/init_64.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/sparc/mm/init_64.c
@@ -2915,7 +2915,7 @@ static void __pte_free(pgtable_t pte)
{
struct ptdesc *ptdesc = virt_to_ptdesc(pte);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
--- a/arch/sparc/mm/srmmu.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/sparc/mm/srmmu.c
@@ -372,7 +372,7 @@ void pte_free(struct mm_struct *mm, pgta
page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT);
spin_lock(&mm->page_table_lock);
if (page_ref_dec_return(page) == 1)
- pagetable_pte_dtor(page_ptdesc(page));
+ pagetable_dtor(page_ptdesc(page));
spin_unlock(&mm->page_table_lock);
srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE);
--- a/arch/um/include/asm/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/um/include/asm/pgalloc.h
@@ -27,7 +27,7 @@ extern pgd_t *pgd_alloc(struct mm_struct
#define __pte_free_tlb(tlb, pte, address) \
do { \
- pagetable_pte_dtor(page_ptdesc(pte)); \
+ pagetable_dtor(page_ptdesc(pte)); \
tlb_remove_page_ptdesc((tlb), (page_ptdesc(pte))); \
} while (0)
@@ -35,7 +35,7 @@ do { \
#define __pmd_free_tlb(tlb, pmd, address) \
do { \
- pagetable_pmd_dtor(virt_to_ptdesc(pmd)); \
+ pagetable_dtor(virt_to_ptdesc(pmd)); \
tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pmd)); \
} while (0)
@@ -43,7 +43,7 @@ do { \
#define __pud_free_tlb(tlb, pud, address) \
do { \
- pagetable_pud_dtor(virt_to_ptdesc(pud)); \
+ pagetable_dtor(virt_to_ptdesc(pud)); \
tlb_remove_page_ptdesc((tlb), virt_to_ptdesc(pud)); \
} while (0)
--- a/arch/x86/mm/pgtable.c~mm-pgtable-introduce-pagetable_dtor
+++ a/arch/x86/mm/pgtable.c
@@ -60,7 +60,7 @@ early_param("userpte", setup_userpte);
void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
{
- pagetable_pte_dtor(page_ptdesc(pte));
+ pagetable_dtor(page_ptdesc(pte));
paravirt_release_pte(page_to_pfn(pte));
paravirt_tlb_remove_table(tlb, pte);
}
@@ -77,7 +77,7 @@ void ___pmd_free_tlb(struct mmu_gather *
#ifdef CONFIG_X86_PAE
tlb->need_flush_all = 1;
#endif
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc));
}
@@ -86,7 +86,7 @@ void ___pud_free_tlb(struct mmu_gather *
{
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
- pagetable_pud_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
paravirt_tlb_remove_table(tlb, virt_to_page(pud));
}
@@ -96,7 +96,7 @@ void ___p4d_free_tlb(struct mmu_gather *
{
struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
- pagetable_p4d_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT);
paravirt_tlb_remove_table(tlb, virt_to_page(p4d));
}
@@ -233,7 +233,7 @@ static void free_pmds(struct mm_struct *
if (pmds[i]) {
ptdesc = virt_to_ptdesc(pmds[i]);
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
mm_dec_nr_pmds(mm);
}
@@ -867,7 +867,7 @@ int pud_free_pmd_page(pud_t *pud, unsign
free_page((unsigned long)pmd_sv);
- pagetable_pmd_dtor(virt_to_ptdesc(pmd));
+ pagetable_dtor(virt_to_ptdesc(pmd));
free_page((unsigned long)pmd);
return 1;
--- a/Documentation/mm/split_page_table_lock.rst~mm-pgtable-introduce-pagetable_dtor
+++ a/Documentation/mm/split_page_table_lock.rst
@@ -62,7 +62,7 @@ Support of split page table lock by an a
===================================================
There's no need in special enabling of PTE split page table lock: everything
-required is done by pagetable_pte_ctor() and pagetable_pte_dtor(), which
+required is done by pagetable_pte_ctor() and pagetable_dtor(), which
must be called on PTE table allocation / freeing.
Make sure the architecture doesn't use slab allocator for page table
@@ -73,7 +73,7 @@ PMD split lock only makes sense if you h
levels.
PMD split lock enabling requires pagetable_pmd_ctor() call on PMD table
-allocation and pagetable_pmd_dtor() on freeing.
+allocation and pagetable_dtor() on freeing.
Allocation usually happens in pmd_alloc_one(), freeing in pmd_free() and
pmd_free_tlb(), but make sure you cover all PMD table allocation / freeing
--- a/include/asm-generic/pgalloc.h~mm-pgtable-introduce-pagetable_dtor
+++ a/include/asm-generic/pgalloc.h
@@ -109,7 +109,7 @@ static inline void pte_free(struct mm_st
{
struct ptdesc *ptdesc = page_ptdesc(pte_page);
- pagetable_pte_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
@@ -153,7 +153,7 @@ static inline void pmd_free(struct mm_st
struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
- pagetable_pmd_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
#endif
@@ -202,7 +202,7 @@ static inline void __pud_free(struct mm_
struct ptdesc *ptdesc = virt_to_ptdesc(pud);
BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
- pagetable_pud_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
@@ -248,7 +248,7 @@ static inline void __p4d_free(struct mm_
struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
- pagetable_p4d_dtor(ptdesc);
+ pagetable_dtor(ptdesc);
pagetable_free(ptdesc);
}
--- a/include/linux/mm.h~mm-pgtable-introduce-pagetable_dtor
+++ a/include/linux/mm.h
@@ -2992,6 +2992,15 @@ static inline bool ptlock_init(struct pt
static inline void ptlock_free(struct ptdesc *ptdesc) {}
#endif /* defined(CONFIG_SPLIT_PTE_PTLOCKS) */
+static inline void pagetable_dtor(struct ptdesc *ptdesc)
+{
+ struct folio *folio = ptdesc_folio(ptdesc);
+
+ ptlock_free(ptdesc);
+ __folio_clear_pgtable(folio);
+ lruvec_stat_sub_folio(folio, NR_PAGETABLE);
+}
+
static inline bool pagetable_pte_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
@@ -3003,15 +3012,6 @@ static inline bool pagetable_pte_ctor(st
return true;
}
-static inline void pagetable_pte_dtor(struct ptdesc *ptdesc)
-{
- struct folio *folio = ptdesc_folio(ptdesc);
-
- ptlock_free(ptdesc);
- __folio_clear_pgtable(folio);
- lruvec_stat_sub_folio(folio, NR_PAGETABLE);
-}
-
pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
static inline pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr,
pmd_t *pmdvalp)
@@ -3088,14 +3088,6 @@ static inline bool pmd_ptlock_init(struc
return ptlock_init(ptdesc);
}
-static inline void pmd_ptlock_free(struct ptdesc *ptdesc)
-{
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
- VM_BUG_ON_PAGE(ptdesc->pmd_huge_pte, ptdesc_page(ptdesc));
-#endif
- ptlock_free(ptdesc);
-}
-
#define pmd_huge_pte(mm, pmd) (pmd_ptdesc(pmd)->pmd_huge_pte)
#else
@@ -3106,7 +3098,6 @@ static inline spinlock_t *pmd_lockptr(st
}
static inline bool pmd_ptlock_init(struct ptdesc *ptdesc) { return true; }
-static inline void pmd_ptlock_free(struct ptdesc *ptdesc) {}
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
@@ -3131,15 +3122,6 @@ static inline bool pagetable_pmd_ctor(st
return true;
}
-static inline void pagetable_pmd_dtor(struct ptdesc *ptdesc)
-{
- struct folio *folio = ptdesc_folio(ptdesc);
-
- pmd_ptlock_free(ptdesc);
- __folio_clear_pgtable(folio);
- lruvec_stat_sub_folio(folio, NR_PAGETABLE);
-}
-
/*
* No scalability reason to split PUD locks yet, but follow the same pattern
* as the PMD locks to make it easier if we decide to. The VM should not be
@@ -3167,14 +3149,6 @@ static inline void pagetable_pud_ctor(st
lruvec_stat_add_folio(folio, NR_PAGETABLE);
}
-static inline void pagetable_pud_dtor(struct ptdesc *ptdesc)
-{
- struct folio *folio = ptdesc_folio(ptdesc);
-
- __folio_clear_pgtable(folio);
- lruvec_stat_sub_folio(folio, NR_PAGETABLE);
-}
-
static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
{
struct folio *folio = ptdesc_folio(ptdesc);
@@ -3183,14 +3157,6 @@ static inline void pagetable_p4d_ctor(st
lruvec_stat_add_folio(folio, NR_PAGETABLE);
}
-static inline void pagetable_p4d_dtor(struct ptdesc *ptdesc)
-{
- struct folio *folio = ptdesc_folio(ptdesc);
-
- __folio_clear_pgtable(folio);
- lruvec_stat_sub_folio(folio, NR_PAGETABLE);
-}
-
extern void __init pagecache_init(void);
extern void free_initmem(void);
--- a/mm/memory.c~mm-pgtable-introduce-pagetable_dtor
+++ a/mm/memory.c
@@ -7031,7 +7031,8 @@ bool ptlock_alloc(struct ptdesc *ptdesc)
void ptlock_free(struct ptdesc *ptdesc)
{
- kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
+ if (ptdesc->ptl)
+ kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
}
#endif
_
Patches currently in -mm which might be from zhengqi.arch@xxxxxxxxxxxxx are
mm-pgtable-make-ptep_clear-non-atomic.patch
mm-khugepaged-recheck-pmd-state-in-retract_page_tables.patch
mm-userfaultfd-recheck-dst_pmd-entry-in-move_pages_pte.patch
mm-userfaultfd-recheck-dst_pmd-entry-in-move_pages_pte-fix.patch
mm-introduce-zap_nonpresent_ptes.patch
mm-introduce-do_zap_pte_range.patch
mm-skip-over-all-consecutive-none-ptes-in-do_zap_pte_range.patch
mm-zap_install_uffd_wp_if_needed-return-whether-uffd-wp-pte-has-been-re-installed.patch
mm-do_zap_pte_range-return-any_skipped-information-to-the-caller.patch
mm-make-zap_pte_range-handle-full-within-pmd-range.patch
mm-pgtable-reclaim-empty-pte-page-in-madvisemadv_dontneed.patch
mm-pgtable-reclaim-empty-pte-page-in-madvisemadv_dontneed-fix.patch
x86-mm-free-page-table-pages-by-rcu-instead-of-semi-rcu.patch
mm-pgtable-make-ptlock-be-freed-by-rcu.patch
x86-select-arch_supports_pt_reclaim-if-x86_64.patch
revert-mm-pgtable-make-ptlock-be-freed-by-rcu.patch
mm-pgtable-add-statistics-for-p4d-level-page-table.patch
arm64-pgtable-use-mmu-gather-to-free-p4d-level-page-table.patch
s390-pgtable-add-statistics-for-pud-and-p4d-level-page-table.patch
mm-pgtable-introduce-pagetable_dtor.patch
arm-pgtable-move-pagetable_dtor-to-__tlb_remove_table.patch
arm64-pgtable-move-pagetable_dtor-to-__tlb_remove_table.patch
riscv-pgtable-move-pagetable_dtor-to-__tlb_remove_table.patch
x86-pgtable-move-pagetable_dtor-to-__tlb_remove_table.patch
s390-pgtable-also-move-pagetable_dtor-of-pxd-to-__tlb_remove_table.patch
mm-pgtable-introduce-generic-__tlb_remove_table.patch
mm-pgtable-move-__tlb_remove_table_one-in-x86-to-generic-file.patch
mm-pgtable-remove-tlb_remove_page_ptdesc.patch
mm-pgtable-remove-tlb_remove_ptdesc.patch
mm-pgtable-introduce-generic-pagetable_dtor_free.patch
