Now the PUD level ptes are always protected by mm->page_table_lock,
which means no split pagetable lock needed. So the generic PUD level
pagetable pages allocation will not call pgtable_pte_page_ctor/dtor(),
that means we will miss to account PUD level pagetable pages.
So introducing pgtable_pud_page_ctor/dtor(), which are just wrappers
of page_{set,clear}_pgtable() to help to get an accurate PUD pagetable
accounting, when allocating or freeing PUD level pagetable pages.
Moreover this patch will also mark the PUD level pagetable with PG_table
flag, which will help to do sanity validation in unpoison_memory() and
get more accurate pagetable accounting by /proc/kpageflags interface.
Meanwhile converting the architectures with using generic PUD pagatable
allocation to add corresponding pgtable_pud_page_ctor() or
pgtable_pud_page_dtor() to account PUD level pagetable.
Signed-off-by: Baolin Wang <baolin.wang@xxxxxxxxxxxxxxxxx>
---
arch/arm64/include/asm/tlb.h | 5 ++++-
arch/loongarch/include/asm/pgalloc.h | 12 +++++++++---
arch/mips/include/asm/pgalloc.h | 12 +++++++++---
arch/x86/mm/pgtable.c | 5 ++++-
include/asm-generic/pgalloc.h | 12 ++++++++++--
include/linux/mm.h | 10 ++++++++++
6 files changed, 46 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index c995d1f..6665f33 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -94,7 +94,10 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
unsigned long addr)
{
- tlb_remove_table(tlb, virt_to_page(pudp));
+ struct page *page = virt_to_page(pudp);
+
+ pgtable_pud_page_dtor(page);
+ tlb_remove_table(tlb, page);
}
#endif
diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h
index 4bfeb3c..8138101 100644
--- a/arch/loongarch/include/asm/pgalloc.h
+++ b/arch/loongarch/include/asm/pgalloc.h
@@ -89,10 +89,16 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
{
pud_t *pud;
+ struct page *page;
+
+ page = alloc_page(GFP_KERNEL);
+ if (!page)
+ return NULL;
+
+ pgtable_pud_page_ctor(page);
+ pud = (pud_t *)page_address(page);
+ pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
- pud = (pud_t *) __get_free_page(GFP_KERNEL);
- if (pud)
- pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
return pud;
}
diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h
index 7960357..5da5880 100644
--- a/arch/mips/include/asm/pgalloc.h
+++ b/arch/mips/include/asm/pgalloc.h
@@ -89,11 +89,17 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
{
+ struct page *page;
pud_t *pud;
- pud = (pud_t *) __get_free_pages(GFP_KERNEL, PUD_TABLE_ORDER);
- if (pud)
- pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
+ page = alloc_pages(GFP_KERNEL, PUD_TABLE_ORDER);
+ if (!page)
+ return NULL;
+
+ pgtable_pud_page_ctor(page);
+ pud = (pud_t *)page_address(page);
+ pud_init((unsigned long)pud, (unsigned long)invalid_pmd_table);
+
return pud;
}
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index a932d77..ea39670 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -76,8 +76,11 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
#if CONFIG_PGTABLE_LEVELS > 3
void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
{
+ struct page *page = virt_to_page(pud);
+
+ pgtable_pud_page_dtor(page);
paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
- paravirt_tlb_remove_table(tlb, virt_to_page(pud));
+ paravirt_tlb_remove_table(tlb, page);
}
#if CONFIG_PGTABLE_LEVELS > 4
diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index 977bea1..8ce8d7c 100644
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -149,11 +149,16 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
+ struct page *page;
gfp_t gfp = GFP_PGTABLE_USER;
if (mm == &init_mm)
gfp = GFP_PGTABLE_KERNEL;
- return (pud_t *)get_zeroed_page(gfp);
+ page = alloc_pages(gfp, 0);
+ if (!page)
+ return NULL;
+ pgtable_pud_page_ctor(page);
+ return (pud_t *)page_address(page);
}
#ifndef __HAVE_ARCH_PUD_ALLOC_ONE
@@ -174,8 +179,11 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
{
+ struct page *page = virt_to_page(pud);
+
BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
- free_page((unsigned long)pud);
+ pgtable_pud_page_dtor(page);
+ __free_page(page);
}
#ifndef __HAVE_ARCH_PUD_FREE
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 7894bc5..54ed6f7 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2364,6 +2364,16 @@ static inline void page_clear_pgtable(struct page *page)
dec_lruvec_page_state(page, NR_PAGETABLE);
}
+static inline void pgtable_pud_page_ctor(struct page *page)
+{
+ page_set_pgtable(page);
+}
+
+static inline void pgtable_pud_page_dtor(struct page *page)
+{
+ page_clear_pgtable(page);
+}
+
static inline bool pgtable_pte_page_ctor(struct page *page)
{
if (!ptlock_init(page))
--
1.8.3.1
