Unlike x86, ARM always enables virtual memory so it always switches early from
phys_alloc to vmalloc. The changes mostly involve using alloc_page directly
whenever physical addresses are needed, and of course implementing the
architecture-dependent callbacks that vmalloc needs.
The 32-bit root page table doesn't need a full page, but it is simpler to just
allocate one. With split files 32-bit and 64-bit, we could just place
mmu_idmap in .bss.
Signed-off-by: Paolo Bonzini <pbonzini@xxxxxxxxxx>
---
arm/Makefile.common | 3 +++
arm/sieve.c | 1 +
lib/arm/asm/mmu-api.h | 1 -
lib/arm/asm/pgtable.h | 10 +++++---
lib/arm/mmu.c | 67 ++++++++++++++++++++++++++++++++++++++-----------
lib/arm/setup.c | 2 +-
lib/arm64/asm/pgtable.h | 5 ++--
lib/virtio-mmio.c | 4 ++-
8 files changed, 70 insertions(+), 23 deletions(-)
create mode 120000 arm/sieve.c
diff --git a/arm/Makefile.common b/arm/Makefile.common
index e09c3e2..0a039cf 100644
--- a/arm/Makefile.common
+++ b/arm/Makefile.common
@@ -15,6 +15,7 @@ tests-common += $(TEST_DIR)/pci-test.flat
tests-common += $(TEST_DIR)/pmu.flat
tests-common += $(TEST_DIR)/gic.flat
tests-common += $(TEST_DIR)/psci.flat
+tests-common += $(TEST_DIR)/sieve.flat
tests-all = $(tests-common) $(tests)
all: directories $(tests-all)
@@ -35,6 +36,8 @@ include $(SRCDIR)/scripts/asm-offsets.mak
cflatobjs += lib/util.o
cflatobjs += lib/alloc_phys.o
+cflatobjs += lib/alloc_page.o
+cflatobjs += lib/vmalloc.o
cflatobjs += lib/alloc.o
cflatobjs += lib/devicetree.o
cflatobjs += lib/pci.o
diff --git a/arm/sieve.c b/arm/sieve.c
new file mode 120000
index 0000000..8f14a5c
--- /dev/null
+++ b/arm/sieve.c
@@ -0,0 +1 @@
+../x86/sieve.c
\ No newline at end of file
diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
index 623f77f..df3ccf7 100644
--- a/lib/arm/asm/mmu-api.h
+++ b/lib/arm/asm/mmu-api.h
@@ -15,7 +15,6 @@ extern void mmu_mark_enabled(int cpu);
extern void mmu_mark_disabled(int cpu);
extern void mmu_enable(pgd_t *pgtable);
extern void mmu_disable(void);
-extern void mmu_enable_idmap(void);
extern void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
phys_addr_t phys_start, phys_addr_t phys_end,
diff --git a/lib/arm/asm/pgtable.h b/lib/arm/asm/pgtable.h
index 27603db..a95e630 100644
--- a/lib/arm/asm/pgtable.h
+++ b/lib/arm/asm/pgtable.h
@@ -40,10 +40,11 @@ static inline pmd_t *pgd_page_vaddr(pgd_t pgd)
#define pmd_offset(pgd, addr) \
(pgd_page_vaddr(*(pgd)) + pmd_index(addr))
-#define pmd_free(pmd) free(pmd)
+#define pmd_free(pmd) free_page(pmd)
static inline pmd_t *pmd_alloc_one(void)
{
- pmd_t *pmd = memalign(PAGE_SIZE, PTRS_PER_PMD * sizeof(pmd_t));
+ assert(PTRS_PER_PMD * sizeof(pmd_t) == PAGE_SIZE);
+ pmd_t *pmd = alloc_page();
memset(pmd, 0, PTRS_PER_PMD * sizeof(pmd_t));
return pmd;
}
@@ -66,10 +67,11 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
#define pte_offset(pmd, addr) \
(pmd_page_vaddr(*(pmd)) + pte_index(addr))
-#define pte_free(pte) free(pte)
+#define pte_free(pte) free_page(pte)
static inline pte_t *pte_alloc_one(void)
{
- pte_t *pte = memalign(PAGE_SIZE, PTRS_PER_PTE * sizeof(pte_t));
+ assert(PTRS_PER_PTE * sizeof(pte_t) == PAGE_SIZE);
+ pte_t *pte = alloc_page();
memset(pte, 0, PTRS_PER_PTE * sizeof(pte_t));
return pte;
}
diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
index f575dac..21bcf3a 100644
--- a/lib/arm/mmu.c
+++ b/lib/arm/mmu.c
@@ -13,6 +13,8 @@
#include <asm/page.h>
#include "alloc.h"
+#include "alloc_page.h"
+#include "vmalloc.h"
#include <asm/pgtable-hwdef.h>
#include <asm/pgtable.h>
@@ -71,6 +73,43 @@ void mmu_disable(void)
asm_mmu_disable();
}
+static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
+{
+ pgd_t *pgd = pgd_offset(pgtable, vaddr);
+ pmd_t *pmd = pmd_alloc(pgd, vaddr);
+ pte_t *pte = pte_alloc(pmd, vaddr);
+
+ return &pte_val(*pte);
+}
+
+static pteval_t *install_pte(pgd_t *pgtable, uintptr_t vaddr, pteval_t pte)
+{
+ pteval_t *p_pte = get_pte(pgtable, vaddr);
+ *p_pte = pte;
+ return p_pte;
+}
+
+static pteval_t *install_page_prot(pgd_t *pgtable, phys_addr_t phys,
+ uintptr_t vaddr, pgprot_t prot)
+{
+ pteval_t pte = phys;
+ pte |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED;
+ pte |= pgprot_val(prot);
+ return install_pte(pgtable, vaddr, pte);
+}
+
+pteval_t *install_page(pgd_t *pgtable, phys_addr_t phys, void *virt)
+{
+ return install_page_prot(pgtable, phys, (uintptr_t)virt,
+ __pgprot(PTE_WBWA | PTE_USER));
+}
+
+phys_addr_t virt_to_pte_phys(pgd_t *pgtable, void *mem)
+{
+ return (*get_pte(pgtable, (uintptr_t)mem) & PHYS_MASK & -PAGE_SIZE)
+ + ((ulong)mem & (PAGE_SIZE - 1));
+}
+
void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset,
phys_addr_t phys_start, phys_addr_t phys_end,
pgprot_t prot)
@@ -79,15 +118,8 @@ void mmu_set_range_ptes(pgd_t *pgtable, uintptr_t virt_offset,
uintptr_t vaddr = virt_offset & PAGE_MASK;
uintptr_t virt_end = phys_end - paddr + vaddr;
- for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE) {
- pgd_t *pgd = pgd_offset(pgtable, vaddr);
- pmd_t *pmd = pmd_alloc(pgd, vaddr);
- pte_t *pte = pte_alloc(pmd, vaddr);
-
- pte_val(*pte) = paddr;
- pte_val(*pte) |= PTE_TYPE_PAGE | PTE_AF | PTE_SHARED;
- pte_val(*pte) |= pgprot_val(prot);
- }
+ for (; vaddr < virt_end; vaddr += PAGE_SIZE, paddr += PAGE_SIZE)
+ install_page_prot(pgtable, paddr, vaddr, prot);
}
void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
@@ -107,14 +139,20 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
}
-
-void mmu_enable_idmap(void)
+void *setup_mmu(phys_addr_t phys_end)
{
- uintptr_t phys_end = sizeof(long) == 8 || !(PHYS_END >> 32)
- ? PHYS_END : 0xfffff000;
uintptr_t code_end = (uintptr_t)&etext;
- mmu_idmap = pgd_alloc();
+ /* 0G-1G = I/O, 1G-3G = identity, 3G-4G = vmalloc */
+ if (phys_end > (3ul << 30))
+ phys_end = 3ul << 30;
+
+#ifdef __aarch64__
+ init_alloc_vpage((void*)(4ul << 30));
+#endif
+
+ mmu_idmap = alloc_page();
+ memset(mmu_idmap, 0, PAGE_SIZE);
mmu_set_range_sect(mmu_idmap, PHYS_IO_OFFSET,
PHYS_IO_OFFSET, PHYS_IO_END,
@@ -130,4 +168,5 @@ void mmu_enable_idmap(void)
__pgprot(PTE_WBWA | PTE_USER));
mmu_enable(mmu_idmap);
+ return mmu_idmap;
}
diff --git a/lib/arm/setup.c b/lib/arm/setup.c
index 0e2bab4..326e78f 100644
--- a/lib/arm/setup.c
+++ b/lib/arm/setup.c
@@ -108,7 +108,7 @@ static void mem_init(phys_addr_t freemem_start)
phys_alloc_init(freemem_start, primary.end - freemem_start);
phys_alloc_set_minimum_alignment(SMP_CACHE_BYTES);
- mmu_enable_idmap();
+ setup_vm();
}
void setup(const void *fdt)
diff --git a/lib/arm64/asm/pgtable.h b/lib/arm64/asm/pgtable.h
index 14d7e3c..941a850 100644
--- a/lib/arm64/asm/pgtable.h
+++ b/lib/arm64/asm/pgtable.h
@@ -48,10 +48,11 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
#define pte_offset(pmd, addr) \
(pmd_page_vaddr(*(pmd)) + pte_index(addr))
-#define pte_free(pte) free(pte)
+#define pte_free(pte) free_page(pte)
static inline pte_t *pte_alloc_one(void)
{
- pte_t *pte = memalign(PAGE_SIZE, PTRS_PER_PTE * sizeof(pte_t));
+ assert(PTRS_PER_PTE * sizeof(pte_t) == PAGE_SIZE);
+ pte_t *pte = alloc_page();
memset(pte, 0, PTRS_PER_PTE * sizeof(pte_t));
return pte;
}
diff --git a/lib/virtio-mmio.c b/lib/virtio-mmio.c
index e4a92f1..e5e8f66 100644
--- a/lib/virtio-mmio.c
+++ b/lib/virtio-mmio.c
@@ -7,6 +7,7 @@
*/
#include "libcflat.h"
#include "devicetree.h"
+#include "alloc_page.h"
#include "alloc.h"
#include "asm/page.h"
#include "asm/io.h"
@@ -53,7 +54,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev,
unsigned num = VIRTIO_MMIO_QUEUE_NUM_MIN;
vq = calloc(1, sizeof(*vq));
- queue = memalign(PAGE_SIZE, VIRTIO_MMIO_QUEUE_SIZE_MIN);
+ assert(VIRTIO_MMIO_QUEUE_SIZE_MIN <= 2*PAGE_SIZE);
+ queue = alloc_pages(1);
assert(vq && queue);
writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
--
2.14.2
