On some architectures (e.g. x86_64 and arm64), vmemmap pages are usually
mapped with huge pmd. We will disable the huge pmd mapping of vmemmap
pages when the feature of "Free vmemmap pages of HugeTLB page" is enabled.
This can affect the non-HugeTLB pages. What we want is only mapping the
vmemmap pages associated with HugeTLB pages with base page. We can split
the huge pmd mapping of vmemmap pages when freeing vmemmap pages of
HugeTLB page. But we need to preallocate page tables. In this patch, we
introduce page tables allocationg/freeing helpers.
Signed-off-by: Muchun Song <songmuchun@xxxxxxxxxxxxx>
---
mm/hugetlb_vmemmap.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++
mm/hugetlb_vmemmap.h | 12 ++++++++++++
2 files changed, 66 insertions(+)
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index f9f9bb212319..628e2752714f 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -170,6 +170,9 @@
*/
#define pr_fmt(fmt) "HugeTLB: " fmt
+#include <linux/list.h>
+#include <asm/pgalloc.h>
+
#include "hugetlb_vmemmap.h"
/*
@@ -209,6 +212,57 @@ static inline unsigned long free_vmemmap_pages_size_per_hpage(struct hstate *h)
return (unsigned long)free_vmemmap_pages_per_hpage(h) << PAGE_SHIFT;
}
+static inline unsigned int vmemmap_pages_per_hpage(struct hstate *h)
+{
+ return free_vmemmap_pages_per_hpage(h) + RESERVE_VMEMMAP_NR;
+}
+
+static inline unsigned long vmemmap_pages_size_per_hpage(struct hstate *h)
+{
+ return (unsigned long)vmemmap_pages_per_hpage(h) << PAGE_SHIFT;
+}
+
+static inline unsigned int pgtable_pages_to_prealloc_per_hpage(struct hstate *h)
+{
+ unsigned long vmemmap_size = vmemmap_pages_size_per_hpage(h);
+
+ /*
+ * No need to pre-allocate page tables when there is no vmemmap pages
+ * to be freed.
+ */
+ if (!free_vmemmap_pages_per_hpage(h))
+ return 0;
+
+ return ALIGN(vmemmap_size, PMD_SIZE) >> PMD_SHIFT;
+}
+
+void vmemmap_pgtable_free(struct list_head *pgtables)
+{
+ struct page *pte_page, *t_page;
+
+ list_for_each_entry_safe(pte_page, t_page, pgtables, lru)
+ pte_free_kernel(&init_mm, page_to_virt(pte_page));
+}
+
+int vmemmap_pgtable_prealloc(struct hstate *h, struct list_head *pgtables)
+{
+ unsigned int nr = pgtable_pages_to_prealloc_per_hpage(h);
+
+ while (nr--) {
+ pte_t *pte_p;
+
+ pte_p = pte_alloc_one_kernel(&init_mm);
+ if (!pte_p)
+ goto out;
+ list_add(&virt_to_page(pte_p)->lru, pgtables);
+ }
+
+ return 0;
+out:
+ vmemmap_pgtable_free(pgtables);
+ return -ENOMEM;
+}
+
/*
* Previously discarded vmemmap pages will be allocated and remapping
* after this function returns zero.
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index cb2bef8f9e73..306e15519da1 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -14,6 +14,8 @@
int alloc_huge_page_vmemmap(struct hstate *h, struct page *head);
void free_huge_page_vmemmap(struct hstate *h, struct page *head);
void hugetlb_vmemmap_init(struct hstate *h);
+int vmemmap_pgtable_prealloc(struct hstate *h, struct list_head *pgtables);
+void vmemmap_pgtable_free(struct list_head *pgtables);
/*
* How many vmemmap pages associated with a HugeTLB page that can be freed
@@ -33,6 +35,16 @@ static inline void free_huge_page_vmemmap(struct hstate *h, struct page *head)
{
}
+static inline int vmemmap_pgtable_prealloc(struct hstate *h,
+ struct list_head *pgtables)
+{
+ return 0;
+}
+
+static inline void vmemmap_pgtable_free(struct list_head *pgtables)
+{
+}
+
static inline void hugetlb_vmemmap_init(struct hstate *h)
{
}
--
2.11.0
