Rework the HYP idmap code to be more consistent with what the kernel
does:
- Move the HYP pgd to the core code (so it can benefit to other
hypervisors)
- Populate this pgd with an identity mapping of the code contained
in the .hyp_idmap.text section
- Offer a method to drop the this identity mapping
- Make all the above depend on CONFIG_ARM_VIRT_EXT
This makes the core code more generic, and remove some of the
clutter from the KVM init.
Cc: Will Deacon <will.deacon at arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier at arm.com>
---
This is now part of my kvm-arm-mm-3.5-rc0 branch. My hope is that
it would make the idmap hook more acceptable by the ARM maintainer.
arch/arm/include/asm/pgtable.h | 7 +++--
arch/arm/kernel/vmlinux.lds.S | 7 ++++++
arch/arm/kvm/arm.c | 31 +++++---------------------
arch/arm/kvm/init.S | 3 ++
arch/arm/kvm/mmu.c | 25 +-------------------
arch/arm/mm/idmap.c | 47 ++++++++++++++++++++++++++++++++-------
6 files changed, 60 insertions(+), 60 deletions(-)
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index fffc01f..9f549c6 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -315,9 +315,10 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
#define pgtable_cache_init() do { } while (0)
-#ifdef CONFIG_KVM_ARM_HOST
-void hyp_idmap_add(pgd_t *, unsigned long, unsigned long);
-void hyp_idmap_del(pgd_t *pgd, unsigned long addr, unsigned long end);
+#ifdef CONFIG_ARM_VIRT_EXT
+extern pgd_t *hyp_pgd;
+
+void hyp_idmap_teardown(void);
#endif
#endif /* !__ASSEMBLY__ */
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index 43a31fb..6bfd6ea 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -21,6 +21,12 @@
*(.idmap.text) \
VMLINUX_SYMBOL(__idmap_text_end) = .;
+#define HYP_IDMAP_TEXT \
+ . = ALIGN(PAGE_SIZE); \
+ VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \
+ *(.hyp_idmap.text) \
+ VMLINUX_SYMBOL(__hyp_idmap_text_end) = .;
+
#ifdef CONFIG_HOTPLUG_CPU
#define ARM_CPU_DISCARD(x)
#define ARM_CPU_KEEP(x) x
@@ -101,6 +107,7 @@ SECTIONS
LOCK_TEXT
KPROBES_TEXT
IDMAP_TEXT
+ HYP_IDMAP_TEXT
#ifdef CONFIG_MMU
*(.fixup)
#endif
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index a73c613..a9687a7 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -627,7 +627,7 @@ static void cpu_init_hyp_mode(void *vector)
cpu_set_vector(vector);
- pgd_ptr = virt_to_phys(kvm_hyp_pgd_get());
+ pgd_ptr = virt_to_phys(hyp_pgd);
stack_page = __get_cpu_var(kvm_arm_hyp_stack_page);
hyp_stack_ptr = stack_page + PAGE_SIZE;
@@ -648,7 +648,7 @@ static void cpu_init_hyp_mode(void *vector)
*/
static int init_hyp_mode(void)
{
- phys_addr_t init_phys_addr, init_end_phys_addr;
+ phys_addr_t init_phys_addr;
int cpu;
int err = 0;
@@ -670,30 +670,15 @@ static int init_hyp_mode(void)
}
/*
- * Allocate Hyp level-1 page table
- */
- err = kvm_hyp_pgd_alloc();
- if (err)
- goto out_free_stack_pages;
-
- init_phys_addr = virt_to_phys(__kvm_hyp_init);
- init_end_phys_addr = virt_to_phys(__kvm_hyp_init_end);
- BUG_ON(init_phys_addr & 0x1f);
-
- /*
- * Create identity mapping for the init code.
- */
- hyp_idmap_add(kvm_hyp_pgd_get(),
- (unsigned long)init_phys_addr,
- (unsigned long)init_end_phys_addr);
-
- /*
* Execute the init code on each CPU.
*
* Note: The stack is not mapped yet, so don't do anything else than
* initializing the hypervisor mode on each CPU using a local stack
* space for temporary storage.
*/
+ init_phys_addr = virt_to_phys(__kvm_hyp_init);
+ BUG_ON(init_phys_addr & 0x1f);
+
for_each_online_cpu(cpu) {
smp_call_function_single(cpu, cpu_init_hyp_mode,
(void *)(long)init_phys_addr, 1);
@@ -702,9 +687,7 @@ static int init_hyp_mode(void)
/*
* Unmap the identity mapping
*/
- hyp_idmap_del(kvm_hyp_pgd_get(),
- (unsigned long)init_phys_addr,
- (unsigned long)init_end_phys_addr);
+ hyp_idmap_teardown();
/*
* Map the Hyp-code called directly from the host
@@ -738,7 +721,6 @@ static int init_hyp_mode(void)
return 0;
out_free_mappings:
free_hyp_pmds();
- kvm_hyp_pgd_free();
out_free_stack_pages:
for_each_possible_cpu(cpu)
free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
@@ -778,7 +760,6 @@ void kvm_arch_exit(void)
free_hyp_pmds();
for_each_possible_cpu(cpu)
free_page(per_cpu(kvm_arm_hyp_stack_page, cpu));
- kvm_hyp_pgd_free();
}
static int arm_init(void)
diff --git a/arch/arm/kvm/init.S b/arch/arm/kvm/init.S
index 0bcc739..b770d78 100644
--- a/arch/arm/kvm/init.S
+++ b/arch/arm/kvm/init.S
@@ -28,6 +28,7 @@
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
.text
.arm
+ .pushsection .hyp_idmap.text,"ax"
.align 12
__kvm_hyp_init:
.globl __kvm_hyp_init
@@ -113,3 +114,5 @@ __do_hyp_init:
__kvm_init_sp:
.globl __kvm_hyp_init_end
__kvm_hyp_init_end:
+
+ .popsection
diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c
index 8d8530f..11faf9a 100644
--- a/arch/arm/kvm/mmu.c
+++ b/arch/arm/kvm/mmu.c
@@ -26,7 +26,6 @@
#include "trace.h"
-static pgd_t *kvm_hyp_pgd;
static DEFINE_MUTEX(kvm_hyp_pgd_mutex);
static void free_ptes(pmd_t *pmd, unsigned long addr)
@@ -58,7 +57,7 @@ void free_hyp_pmds(void)
mutex_lock(&kvm_hyp_pgd_mutex);
for (addr = PAGE_OFFSET; addr != 0; addr += PGDIR_SIZE) {
- pgd = kvm_hyp_pgd + pgd_index(addr);
+ pgd = hyp_pgd + pgd_index(addr);
pud = pud_offset(pgd, addr);
BUG_ON(pud_bad(*pud));
@@ -166,7 +165,7 @@ static int __create_hyp_mappings(void *from, void *to,
mutex_lock(&kvm_hyp_pgd_mutex);
for (addr = start; addr < end; addr = next) {
- pgd = kvm_hyp_pgd + pgd_index(addr);
+ pgd = hyp_pgd + pgd_index(addr);
pud = pud_offset(pgd, addr);
if (pud_none_or_clear_bad(pud)) {
@@ -699,23 +698,3 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
return 0;
}
-
-int kvm_hyp_pgd_alloc(void)
-{
- kvm_hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
- if (!kvm_hyp_pgd)
- return -ENOMEM;
-
- return 0;
-}
-
-pgd_t *kvm_hyp_pgd_get(void)
-{
- return kvm_hyp_pgd;
-}
-
-void kvm_hyp_pgd_free(void)
-{
- kfree(kvm_hyp_pgd);
- kvm_hyp_pgd = NULL;
-}
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 87d00ae..f6c498a 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -1,5 +1,6 @@
#include <linux/module.h>
#include <linux/kernel.h>
+#include <linux/slab.h>
#include <asm/cputype.h>
#include <asm/idmap.h>
@@ -99,12 +100,8 @@ static int __init init_static_idmap(void)
}
early_initcall(init_static_idmap);
-#ifdef CONFIG_KVM_ARM_HOST
-void hyp_idmap_add(pgd_t *pgd, unsigned long addr, unsigned long end)
-{
- identity_mapping_add(pgd, addr, end, PMD_SECT_AP1);
-}
-EXPORT_SYMBOL_GPL(hyp_idmap_add);
+#ifdef CONFIG_ARM_VIRT_EXT
+pgd_t *hyp_pgd;
static void hyp_idmap_del_pmd(pgd_t *pgd, unsigned long addr)
{
@@ -113,15 +110,16 @@ static void hyp_idmap_del_pmd(pgd_t *pgd, unsigned long addr)
pud = pud_offset(pgd, addr);
pmd = pmd_offset(pud, addr);
- pmd_free(NULL, pmd);
pud_clear(pud);
+ clean_pmd_entry(pmd);
+ pmd_free(NULL, (pmd_t *)((unsigned long)pmd & PAGE_MASK));
}
/*
* This version actually frees the underlying pmds for all pgds in range and
* clear the pgds themselves afterwards.
*/
-void hyp_idmap_del(pgd_t *pgd, unsigned long addr, unsigned long end)
+static void hyp_idmap_del(pgd_t *pgd, unsigned long addr, unsigned long end)
{
unsigned long next;
@@ -132,7 +130,38 @@ void hyp_idmap_del(pgd_t *pgd, unsigned long addr, unsigned long end)
hyp_idmap_del_pmd(pgd, addr);
} while (pgd++, addr = next, addr < end);
}
-EXPORT_SYMBOL_GPL(hyp_idmap_del);
+
+extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
+
+void hyp_idmap_teardown(void)
+{
+ phys_addr_t idmap_start, idmap_end;
+
+ idmap_start = virt_to_phys((void *)__hyp_idmap_text_start);
+ idmap_end = virt_to_phys((void *)__hyp_idmap_text_end);
+ hyp_idmap_del(hyp_pgd, idmap_start, idmap_end);
+}
+EXPORT_SYMBOL_GPL(hyp_idmap_teardown);
+
+static int __init hyp_init_static_idmap(void)
+{
+ phys_addr_t idmap_start, idmap_end;
+
+ hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
+ if (!hyp_pgd)
+ return -ENOMEM;
+
+ /* Add an identity mapping for the physical address of the section. */
+ idmap_start = virt_to_phys((void *)__hyp_idmap_text_start);
+ idmap_end = virt_to_phys((void *)__hyp_idmap_text_end);
+
+ pr_info("Setting up static HYP identity map for 0x%llx - 0x%llx\n",
+ (long long)idmap_start, (long long)idmap_end);
+ identity_mapping_add(hyp_pgd, idmap_start, idmap_end, PMD_SECT_AP1);
+
+ return 0;
+}
+early_initcall(hyp_init_static_idmap);
#endif
/*
--
1.7.3.4
