Provide functions to copy page-table entries from the kernel page-table
to an ASI page-table for a specified VA range. These functions are based
on the copy_pxx_range() functions defined in mm/memory.c. A difference
is that a level parameter can be specified to indicate the page-table
level (PGD, P4D, PUD PMD, PTE) at which the copy should be done. Also
functions don't rely on mm or vma, and they don't alter the source
page-table even if an entry is bad. Also the VA range start and size
don't need to be page-aligned.
Signed-off-by: Alexandre Chartre <alexandre.chartre@xxxxxxxxxx>
---
arch/x86/include/asm/asi.h | 4 +
arch/x86/mm/asi_pagetable.c | 205 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 209 insertions(+), 0 deletions(-)
diff --git a/arch/x86/include/asm/asi.h b/arch/x86/include/asm/asi.h
index 3d965e6..19656aa 100644
--- a/arch/x86/include/asm/asi.h
+++ b/arch/x86/include/asm/asi.h
@@ -76,6 +76,10 @@ struct asi_session {
extern bool asi_fault(struct pt_regs *regs, unsigned long error_code,
unsigned long address);
+extern int asi_map_range(struct asi *asi, void *ptr, size_t size,
+ enum page_table_level level);
+extern int asi_map(struct asi *asi, void *ptr, unsigned long size);
+
/*
* Function to exit the current isolation. This is used to abort isolation
* when a task using isolation is scheduled out.
diff --git a/arch/x86/mm/asi_pagetable.c b/arch/x86/mm/asi_pagetable.c
index e17af9e..0169395 100644
--- a/arch/x86/mm/asi_pagetable.c
+++ b/arch/x86/mm/asi_pagetable.c
@@ -394,3 +394,208 @@ static int asi_set_pgd(struct asi *asi, pgd_t *pgd, pgd_t pgd_value)
return 0;
}
+
+static int asi_copy_pte_range(struct asi *asi, pmd_t *dst_pmd, pmd_t *src_pmd,
+ unsigned long addr, unsigned long end)
+{
+ pte_t *src_pte, *dst_pte;
+
+ dst_pte = asi_pte_alloc(asi, dst_pmd, addr);
+ if (IS_ERR(dst_pte))
+ return PTR_ERR(dst_pte);
+
+ addr &= PAGE_MASK;
+ src_pte = pte_offset_map(src_pmd, addr);
+
+ do {
+ asi_set_pte(asi, dst_pte, *src_pte);
+
+ } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr < end);
+
+ return 0;
+}
+
+static int asi_copy_pmd_range(struct asi *asi, pud_t *dst_pud, pud_t *src_pud,
+ unsigned long addr, unsigned long end,
+ enum page_table_level level)
+{
+ pmd_t *src_pmd, *dst_pmd;
+ unsigned long next;
+ int err;
+
+ dst_pmd = asi_pmd_alloc(asi, dst_pud, addr);
+ if (IS_ERR(dst_pmd))
+ return PTR_ERR(dst_pmd);
+
+ src_pmd = pmd_offset(src_pud, addr);
+
+ do {
+ next = pmd_addr_end(addr, end);
+ if (level == PGT_LEVEL_PMD || pmd_none(*src_pmd) ||
+ pmd_trans_huge(*src_pmd) || pmd_devmap(*src_pmd)) {
+ err = asi_set_pmd(asi, dst_pmd, *src_pmd);
+ if (err)
+ return err;
+ continue;
+ }
+
+ if (!pmd_present(*src_pmd)) {
+ pr_warn("ASI %p: PMD not present for [%lx,%lx]\n",
+ asi, addr, next - 1);
+ pmd_clear(dst_pmd);
+ continue;
+ }
+
+ err = asi_copy_pte_range(asi, dst_pmd, src_pmd, addr, next);
+ if (err) {
+ pr_err("ASI %p: PMD error copying PTE addr=%lx next=%lx\n",
+ asi, addr, next);
+ return err;
+ }
+
+ } while (dst_pmd++, src_pmd++, addr = next, addr < end);
+
+ return 0;
+}
+
+static int asi_copy_pud_range(struct asi *asi, p4d_t *dst_p4d, p4d_t *src_p4d,
+ unsigned long addr, unsigned long end,
+ enum page_table_level level)
+{
+ pud_t *src_pud, *dst_pud;
+ unsigned long next;
+ int err;
+
+ dst_pud = asi_pud_alloc(asi, dst_p4d, addr);
+ if (IS_ERR(dst_pud))
+ return PTR_ERR(dst_pud);
+
+ src_pud = pud_offset(src_p4d, addr);
+
+ do {
+ next = pud_addr_end(addr, end);
+ if (level == PGT_LEVEL_PUD || pud_none(*src_pud) ||
+ pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
+ err = asi_set_pud(asi, dst_pud, *src_pud);
+ if (err)
+ return err;
+ continue;
+ }
+
+ err = asi_copy_pmd_range(asi, dst_pud, src_pud, addr, next,
+ level);
+ if (err) {
+ pr_err("ASI %p: PUD error copying PMD addr=%lx next=%lx\n",
+ asi, addr, next);
+ return err;
+ }
+
+ } while (dst_pud++, src_pud++, addr = next, addr < end);
+
+ return 0;
+}
+
+static int asi_copy_p4d_range(struct asi *asi, pgd_t *dst_pgd, pgd_t *src_pgd,
+ unsigned long addr, unsigned long end,
+ enum page_table_level level)
+{
+ p4d_t *src_p4d, *dst_p4d;
+ unsigned long next;
+ int err;
+
+ dst_p4d = asi_p4d_alloc(asi, dst_pgd, addr);
+ if (IS_ERR(dst_p4d))
+ return PTR_ERR(dst_p4d);
+
+ src_p4d = p4d_offset(src_pgd, addr);
+
+ do {
+ next = p4d_addr_end(addr, end);
+ if (level == PGT_LEVEL_P4D || p4d_none(*src_p4d)) {
+ err = asi_set_p4d(asi, dst_p4d, *src_p4d);
+ if (err)
+ return err;
+ continue;
+ }
+
+ err = asi_copy_pud_range(asi, dst_p4d, src_p4d, addr, next,
+ level);
+ if (err) {
+ pr_err("ASI %p: P4D error copying PUD addr=%lx next=%lx\n",
+ asi, addr, next);
+ return err;
+ }
+
+ } while (dst_p4d++, src_p4d++, addr = next, addr < end);
+
+ return 0;
+}
+
+static int asi_copy_pgd_range(struct asi *asi,
+ pgd_t *dst_pagetable, pgd_t *src_pagetable,
+ unsigned long addr, unsigned long end,
+ enum page_table_level level)
+{
+ pgd_t *src_pgd, *dst_pgd;
+ unsigned long next;
+ int err;
+
+ dst_pgd = pgd_offset_pgd(dst_pagetable, addr);
+ src_pgd = pgd_offset_pgd(src_pagetable, addr);
+
+ do {
+ next = pgd_addr_end(addr, end);
+ if (level == PGT_LEVEL_PGD || pgd_none(*src_pgd)) {
+ err = asi_set_pgd(asi, dst_pgd, *src_pgd);
+ if (err)
+ return err;
+ continue;
+ }
+
+ err = asi_copy_p4d_range(asi, dst_pgd, src_pgd, addr, next,
+ level);
+ if (err) {
+ pr_err("ASI %p: PGD error copying P4D addr=%lx next=%lx\n",
+ asi, addr, next);
+ return err;
+ }
+
+ } while (dst_pgd++, src_pgd++, addr = next, addr < end);
+
+ return 0;
+}
+
+/*
+ * Copy page table entries from the current page table (i.e. from the
+ * kernel page table) to the specified ASI page-table. The level
+ * parameter specifies the page-table level (PGD, P4D, PUD PMD, PTE)
+ * at which the copy should be done.
+ */
+int asi_map_range(struct asi *asi, void *ptr, size_t size,
+ enum page_table_level level)
+{
+ unsigned long addr = (unsigned long)ptr;
+ unsigned long end = addr + ((unsigned long)size);
+ unsigned long flags;
+ int err;
+
+ pr_debug("ASI %p: MAP %px/%lx/%d\n", asi, ptr, size, level);
+
+ spin_lock_irqsave(&asi->lock, flags);
+ err = asi_copy_pgd_range(asi, asi->pgd, current->mm->pgd,
+ addr, end, level);
+ spin_unlock_irqrestore(&asi->lock, flags);
+
+ return err;
+}
+EXPORT_SYMBOL(asi_map_range);
+
+/*
+ * Copy page-table PTE entries from the current page-table to the
+ * specified ASI page-table.
+ */
+int asi_map(struct asi *asi, void *ptr, unsigned long size)
+{
+ return asi_map_range(asi, ptr, size, PGT_LEVEL_PTE);
+}
+EXPORT_SYMBOL(asi_map);
--
1.7.1
