Sets up the required registers to run code in HYP-mode from the kernel.
No major controversies, but we should consider how to deal with SMP
support for hypervisor stack page.
By setting the HVBAR the kernel can execute code in Hyp-mode with
the MMU disabled. The HVBAR initially points to initialization code,
which initializes other Hyp-mode registers and enables the MMU
for Hyp-mode. Afterwards, the HVBAR is changed to point to KVM
Hyp vectors used to catch guest faults and to switch to Hyp mode
to perform a world-switch into a KVM guest.
Also provides memory mapping code to map required code pages and data
structures accessed in Hyp mode at the same virtual address as the
host kernel virtual addresses, but which conforms to the architectural
requirements for translations in Hyp mode. This interface is added in
arch/arm/kvm/arm_mmu.c and is comprised of:
- create_hyp_mappings(hyp_pgd, start, end);
- remove_hyp_mappings(hyp_pgd, start, end);
- free_hyp_pmds(pgd_hyp);
See the implementation for more details.
Signed-off-by: Christoffer Dall <c.dall@xxxxxxxxxxxxxxxxxxxxxx>
---
arch/arm/include/asm/kvm_arm.h | 103 +++++++++++++
arch/arm/include/asm/kvm_asm.h | 23 +++
arch/arm/include/asm/kvm_host.h | 1
arch/arm/include/asm/kvm_mmu.h | 40 +++++
arch/arm/include/asm/pgtable-3level-hwdef.h | 5 +
arch/arm/include/asm/pgtable.h | 5 +
arch/arm/kvm/arm.c | 169 +++++++++++++++++++++
arch/arm/kvm/arm_exports.c | 10 +
arch/arm/kvm/arm_init.S | 98 ++++++++++++
arch/arm/kvm/arm_interrupts.S | 30 ++++
arch/arm/kvm/arm_mmu.c | 213 +++++++++++++++++++++++++++
arch/arm/mm/idmap.c | 7 +
mm/memory.c | 1
13 files changed, 704 insertions(+), 1 deletions(-)
create mode 100644 arch/arm/include/asm/kvm_arm.h
create mode 100644 arch/arm/include/asm/kvm_mmu.h
diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h
new file mode 100644
index 0000000..835abd1
--- /dev/null
+++ b/arch/arm/include/asm/kvm_arm.h
@@ -0,0 +1,103 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+#ifndef __KVM_ARM_H__
+#define __KVM_ARM_H__
+
+#include <asm/types.h>
+
+/* Hyp Configuration Register (HCR) bits */
+#define HCR_TGE (1 << 27)
+#define HCR_TVM (1 << 26)
+#define HCR_TTLB (1 << 25)
+#define HCR_TPU (1 << 24)
+#define HCR_TPC (1 << 23)
+#define HCR_TSW (1 << 22)
+#define HCR_TAC (1 << 21)
+#define HCR_TIDCP (1 << 20)
+#define HCR_TSC (1 << 19)
+#define HCR_TID3 (1 << 18)
+#define HCR_TID2 (1 << 17)
+#define HCR_TID1 (1 << 16)
+#define HCR_TID0 (1 << 15)
+#define HCR_TWE (1 << 14)
+#define HCR_TWI (1 << 13)
+#define HCR_DC (1 << 12)
+#define HCR_BSU (3 << 10)
+#define HCR_FB (1 << 9)
+#define HCR_VA (1 << 8)
+#define HCR_VI (1 << 7)
+#define HCR_VF (1 << 6)
+#define HCR_AMO (1 << 5)
+#define HCR_IMO (1 << 4)
+#define HCR_FMO (1 << 3)
+#define HCR_PTW (1 << 2)
+#define HCR_SWIO (1 << 1)
+#define HCR_VM 1
+#define HCR_GUEST_MASK (HCR_TSC | HCR_TWE | HCR_TWI | HCR_VM | HCR_AMO | \
+ HCR_AMO | HCR_IMO | HCR_FMO | HCR_SWIO)
+
+/* Hyp System Control Register (HSCTLR) bits */
+#define HSCTLR_TE (1 << 30)
+#define HSCTLR_EE (1 << 25)
+#define HSCTLR_FI (1 << 21)
+#define HSCTLR_WXN (1 << 19)
+#define HSCTLR_I (1 << 12)
+#define HSCTLR_C (1 << 2)
+#define HSCTLR_A (1 << 1)
+#define HSCTLR_M 1
+#define HSCTLR_MASK (HSCTLR_M | HSCTLR_A | HSCTLR_C | HSCTLR_I | \
+ HSCTLR_WXN | HSCTLR_FI | HSCTLR_EE | HSCTLR_TE)
+
+/* TTBCR and HTCR Registers bits */
+#define TTBCR_EAE (1 << 31)
+#define TTBCR_IMP (1 << 30)
+#define TTBCR_SH1 (3 << 28)
+#define TTBCR_ORGN1 (3 << 26)
+#define TTBCR_IRGN1 (3 << 24)
+#define TTBCR_EPD1 (1 << 23)
+#define TTBCR_A1 (1 << 22)
+#define TTBCR_T1SZ (3 << 16)
+#define TTBCR_SH0 (3 << 12)
+#define TTBCR_ORGN0 (3 << 10)
+#define TTBCR_IRGN0 (3 << 8)
+#define TTBCR_EPD0 (1 << 7)
+#define TTBCR_T0SZ 3
+#define HTCR_MASK (TTBCR_T0SZ | TTBCR_IRGN0 | TTBCR_ORGN0 | TTBCR_SH0)
+
+
+/* Virtualization Translation Control Register (VTCR) bits */
+#define VTCR_SH0 (3 << 12)
+#define VTCR_ORGN0 (3 << 10)
+#define VTCR_IRGN0 (3 << 8)
+#define VTCR_SL0 (3 << 6)
+#define VTCR_S (1 << 4)
+#define VTCR_T0SZ 3
+#define VTCR_MASK (VTCR_SH0 | VTCR_ORGN0 | VTCR_IRGN0 | VTCR_SL0 | \
+ VTCR_S | VTCR_T0SZ | VTCR_MASK)
+#define VTCR_HTCR_SH (VTCR_SH0 | VTCR_ORGN0 | VTCR_IRGN0)
+#define VTCR_SL_L2 0 /* Starting-level: 2 */
+#define VTCR_SL_L1 (1 << 6) /* Starting-level: 1 */
+#define VTCR_GUEST_SL VTCR_SL_L1
+#define VTCR_GUEST_T0SZ 0
+#if VTCR_GUEST_SL == 0
+#define VTTBR_X (14 - VTCR_GUEST_T0SZ)
+#else
+#define VTTBR_X (5 - VTCR_GUEST_T0SZ)
+#endif
+
+
+#endif /* __KVM_ARM_H__ */
diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index c3d4458..78cf8d3 100644
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -24,5 +24,28 @@
#define ARM_EXCEPTION_DATA_ABORT 4
#define ARM_EXCEPTION_IRQ 5
#define ARM_EXCEPTION_FIQ 6
+#define ARM_EXCEPTION_HVC 7
+
+/*
+ * SMC Hypervisor API call numbers
+ */
+#ifdef __ASSEMBLY__
+.equ SMCHYP_HVBAR_W, 0xfffffff0
+#else /* !__ASSEMBLY__ */
+asm(".equ SMCHYP_HVBAR_W, 0xfffffff0");
+#endif /* __ASSEMBLY__ */
+
+#ifndef __ASSEMBLY__
+struct kvm_vcpu;
+
+extern unsigned long __kvm_hyp_init;
+extern unsigned long __kvm_hyp_init_end;
+
+extern unsigned long __kvm_hyp_vector;
+extern unsigned long __kvm_hyp_vector_end;
+
+extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
+extern unsigned long __kvm_vcpu_run_end;
+#endif
#endif /* __ARM_KVM_ASM_H__ */
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index b2fcd8a..6a10467 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -31,6 +31,7 @@ struct kvm_vcpu;
u32 *kvm_vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num, u32 mode);
struct kvm_arch {
+ pgd_t *pgd; /* 1-level 2nd stage table */
};
#define EXCEPTION_NONE 0
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
new file mode 100644
index 0000000..d22aad0
--- /dev/null
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -0,0 +1,40 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+#ifndef __ARM_KVM_MMU_H__
+#define __ARM_KVM_MMU_H__
+
+/*
+ * The architecture supports 40-bit IPA as input to the 2nd stage translations
+ * and PTRS_PER_PGD2 could therefore be 1024.
+ *
+ * To save a bit of memory and to avoid alignment issues we assume 39-bit IPA
+ * for now, but remember that the level-1 table must be aligned to its size.
+ */
+#define PTRS_PER_PGD2 512
+#define PGD2_ORDER get_order(PTRS_PER_PGD2 * sizeof(pgd_t))
+
+extern pgd_t *kvm_hyp_pgd;
+
+int create_hyp_mappings(pgd_t *hyp_pgd,
+ unsigned long start,
+ unsigned long end);
+void remove_hyp_mappings(pgd_t *hyp_pgd,
+ unsigned long start,
+ unsigned long end);
+void free_hyp_pmds(pgd_t *hyp_pgd);
+
+#endif /* __ARM_KVM_MMU_H__ */
diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h
index 8ed298f..7dd1dba 100644
--- a/arch/arm/include/asm/pgtable-3level-hwdef.h
+++ b/arch/arm/include/asm/pgtable-3level-hwdef.h
@@ -32,6 +32,9 @@
#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
#define PMD_BIT4 (_AT(pmdval_t, 0))
#define PMD_DOMAIN(x) (_AT(pmdval_t, 0))
+#define PMD_APTABLE_SHIFT (61)
+#define PMD_APTABLE (_AT(pgdval_t, 3) << PGD_APTABLE_SHIFT)
+#define PMD_PXNTABLE (_AT(pgdval_t, 1) << 59)
/*
* - section
@@ -44,8 +47,10 @@
#ifdef __ASSEMBLY__
/* avoid 'shift count out of range' warning */
#define PMD_SECT_XN (0)
+#define PMD_SECT_PXN (0)
#else
#define PMD_SECT_XN ((pmdval_t)1 << 54)
+#define PMD_SECT_PXN ((pmdval_t)1 << 53)
#endif
#define PMD_SECT_AP_WRITE (_AT(pmdval_t, 0))
#define PMD_SECT_AP_READ (_AT(pmdval_t, 0))
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index da74cd1..db3b6e8 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -88,6 +88,7 @@ extern pgprot_t pgprot_kernel;
#define PAGE_READONLY_EXEC _MOD_PROT(pgprot_user, L_PTE_USER | L_PTE_RDONLY)
#define PAGE_KERNEL _MOD_PROT(pgprot_kernel, L_PTE_XN)
#define PAGE_KERNEL_EXEC pgprot_kernel
+#define PAGE_HYP _MOD_PROT(pgprot_kernel, L_PTE_USER)
#define __PAGE_NONE __pgprot(_L_PTE_DEFAULT | L_PTE_RDONLY | L_PTE_XN)
#define __PAGE_SHARED __pgprot(_L_PTE_DEFAULT | L_PTE_USER | L_PTE_XN)
@@ -223,6 +224,10 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
#ifdef CONFIG_ARM_LPAE
#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
+#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
+ PMD_TYPE_TABLE)
+#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
+ PMD_TYPE_SECT)
#define copy_pmd(pmdpd,pmdps) \
do { \
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 1e7a907..ccfb225 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -31,9 +31,20 @@
#include <asm/uaccess.h>
#include <asm/ptrace.h>
#include <asm/mman.h>
+#include <asm/tlbflush.h>
+#include <asm/kvm_arm.h>
+#include <asm/kvm_asm.h>
+#include <asm/kvm_mmu.h>
#include "debug.h"
+static void *kvm_arm_hyp_stack_page;
+
+/* The VMID used in the VTTBR */
+#define VMID_SIZE (1<<8)
+static DECLARE_BITMAP(kvm_vmids, VMID_SIZE);
+static DEFINE_MUTEX(kvm_vmids_mutex);
+
int kvm_arch_hardware_enable(void *garbage)
{
return 0;
@@ -245,13 +256,171 @@ long kvm_arch_vm_ioctl(struct file *filp,
return -EINVAL;
}
+/**
+ * Inits Hyp-mode on a single CPU
+ */
+static int init_hyp_mode(void)
+{
+ phys_addr_t init_phys_addr, init_end_phys_addr;
+ unsigned long vector_ptr, hyp_stack_ptr;
+ int err = 0;
+
+ /*
+ * Allocate Hyp level-1 page table
+ */
+ kvm_hyp_pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL);
+ if (!kvm_hyp_pgd)
+ return -ENOMEM;
+
+ /*
+ * Allocate stack page for Hypervisor-mode
+ */
+ kvm_arm_hyp_stack_page = (void *)__get_free_page(GFP_KERNEL);
+ if (!kvm_arm_hyp_stack_page) {
+ err = -ENOMEM;
+ goto out_free_pgd;
+ }
+
+ hyp_stack_ptr = (unsigned long)kvm_arm_hyp_stack_page + PAGE_SIZE;
+
+ init_phys_addr = virt_to_phys((void *)&__kvm_hyp_init);
+ init_end_phys_addr = virt_to_phys((void *)&__kvm_hyp_init_end);
+
+ /*
+ * Create identity mapping
+ */
+ hyp_identity_mapping_add(kvm_hyp_pgd,
+ (unsigned long)init_phys_addr,
+ (unsigned long)init_end_phys_addr);
+
+ /*
+ * Set the HVBAR
+ */
+ BUG_ON(init_phys_addr & 0x1f);
+ asm volatile (
+ "mov r0, %[vector_ptr]\n\t"
+ "ldr r7, =SMCHYP_HVBAR_W\n\t"
+ "smc #0\n\t" : :
+ [vector_ptr] "r" ((unsigned long)init_phys_addr) :
+ "r0", "r7");
+
+ /*
+ * Call initialization code
+ */
+ asm volatile (
+ "mov r0, %[pgd_ptr]\n\t"
+ "mov r1, %[stack_ptr]\n\t"
+ "hvc #0\n\t" : :
+ [pgd_ptr] "r" (virt_to_phys(kvm_hyp_pgd)),
+ [stack_ptr] "r" (hyp_stack_ptr) :
+ "r0", "r1");
+
+ /*
+ * Unmap the identity mapping
+ */
+ hyp_identity_mapping_del(kvm_hyp_pgd,
+ (unsigned long)init_phys_addr,
+ (unsigned long)init_end_phys_addr);
+
+ /*
+ * Set the HVBAR to the virtual kernel address
+ */
+ vector_ptr = (unsigned long)&__kvm_hyp_vector;
+ asm volatile (
+ "mov r0, %[vector_ptr]\n\t"
+ "ldr r7, =SMCHYP_HVBAR_W\n\t"
+ "smc #0\n\t" : :
+ [vector_ptr] "r" ((unsigned long)vector_ptr) :
+ "r0", "r7");
+
+ return err;
+out_free_pgd:
+ kfree(kvm_hyp_pgd);
+ kvm_hyp_pgd = NULL;
+ return err;
+}
+
+/*
+ * Initializes the memory mappings used in Hyp-mode
+ *
+ * Code executed in Hyp-mode and a stack page per cpu must be mapped into the
+ * hypervisor translation tables.
+ *
+ * Currently there is no SMP support so we map only a single stack page on a
+ * single CPU.
+ */
+static int init_hyp_memory(void)
+{
+ int err = 0;
+ unsigned long start, end;
+
+ /*
+ * Map Hyp exception vectors
+ */
+ start = (unsigned long)&__kvm_hyp_vector;
+ end = (unsigned long)&__kvm_hyp_vector_end;
+ err = create_hyp_mappings(kvm_hyp_pgd, start, end);
+ if (err) {
+ kvm_err(err, "Cannot map hyp vector");
+ goto out_free_mappings;
+ }
+
+ /*
+ * Map the world-switch code
+ */
+ start = (unsigned long)&__kvm_vcpu_run;
+ end = (unsigned long)&__kvm_vcpu_run_end;
+ err = create_hyp_mappings(kvm_hyp_pgd, start, end);
+ if (err) {
+ kvm_err(err, "Cannot map world-switch code");
+ goto out_free_mappings;
+ }
+
+ /*
+ * Map the Hyp stack page
+ */
+ start = (unsigned long)kvm_arm_hyp_stack_page;
+ end = start + PAGE_SIZE - 1;
+ err = create_hyp_mappings(kvm_hyp_pgd, start, end);
+ if (err) {
+ kvm_err(err, "Cannot map hyp stack");
+ goto out_free_mappings;
+ }
+
+ return err;
+out_free_mappings:
+ free_hyp_pmds(kvm_hyp_pgd);
+ return err;
+}
+
+/**
+ * Initialize Hyp-mode and memory mappings on all CPUs.
+ */
int kvm_arch_init(void *opaque)
{
+ int err;
+
+ err = init_hyp_mode();
+ if (err)
+ goto out_err;
+
+ err = init_hyp_memory();
+ if (err)
+ goto out_err;
+
+ set_bit(0, kvm_vmids);
return 0;
+out_err:
+ return err;
}
void kvm_arch_exit(void)
{
+ if (kvm_hyp_pgd) {
+ free_hyp_pmds(kvm_hyp_pgd);
+ kfree(kvm_hyp_pgd);
+ kvm_hyp_pgd = NULL;
+ }
}
static int arm_init(void)
diff --git a/arch/arm/kvm/arm_exports.c b/arch/arm/kvm/arm_exports.c
index d8a7fd5..0fdd5ff 100644
--- a/arch/arm/kvm/arm_exports.c
+++ b/arch/arm/kvm/arm_exports.c
@@ -14,3 +14,13 @@
*/
#include <linux/module.h>
+#include <asm/kvm_asm.h>
+
+EXPORT_SYMBOL_GPL(__kvm_hyp_init);
+EXPORT_SYMBOL_GPL(__kvm_hyp_init_end);
+
+EXPORT_SYMBOL_GPL(__kvm_hyp_vector);
+EXPORT_SYMBOL_GPL(__kvm_hyp_vector_end);
+
+EXPORT_SYMBOL_GPL(__kvm_vcpu_run);
+EXPORT_SYMBOL_GPL(__kvm_vcpu_run_end);
diff --git a/arch/arm/kvm/arm_init.S b/arch/arm/kvm/arm_init.S
index 073a494..5f7e922 100644
--- a/arch/arm/kvm/arm_init.S
+++ b/arch/arm/kvm/arm_init.S
@@ -13,5 +13,103 @@
* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
+
+#include <linux/linkage.h>
+#include <asm/unified.h>
#include <asm/asm-offsets.h>
#include <asm/kvm_asm.h>
+#include <asm/kvm_arm.h>
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@ Hypervisor initialization
+@ - should be called with:
+@ r0 = Hypervisor pgd pointer
+@ r1 = top of Hyp stack (kernel VA)
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+ .text
+ .arm
+ .align 12
+__kvm_hyp_init:
+ .globl __kvm_hyp_init
+
+ @ Hyp-mode exception vector
+ nop
+ nop
+ nop
+ nop
+ nop
+ b __do_hyp_init
+ nop
+ nop
+
+__do_hyp_init:
+ @ Set the sp to end of this page and push data for later use
+ mov sp, pc
+ bic sp, sp, #0x0ff
+ bic sp, sp, #0xf00
+ add sp, sp, #0x1000
+ push {r1, r2, r12}
+
+ @ Set the HTTBR to point to the hypervisor PGD pointer passed to
+ @ function and set the upper bits equal to the kernel PGD.
+ mrrc p15, 1, r1, r2, c2
+ mcrr p15, 4, r0, r2, c2
+
+ @ Set the HTCR and VTCR to the same shareability and cacheability
+ @ settings as the non-secure TTBCR and with T0SZ == 0.
+ mrc p15, 4, r0, c2, c0, 2 @ HTCR
+ ldr r12, =HTCR_MASK
+ bic r0, r0, r12
+ mrc p15, 0, r1, c2, c0, 2 @ TTBCR
+ and r1, r1, #(HTCR_MASK & ~TTBCR_T0SZ)
+ orr r0, r0, r1
+ mcr p15, 4, r0, c2, c0, 2 @ HTCR
+
+ mrc p15, 4, r1, c2, c1, 2 @ VTCR
+ bic r1, r1, #(VTCR_HTCR_SH | VTCR_SL0)
+ bic r0, r0, #(~VTCR_HTCR_SH)
+ orr r1, r0, r1
+ orr r1, r1, #(VTCR_SL_L1 | VTCR_GUEST_T0SZ)
+ mcr p15, 4, r1, c2, c1, 2 @ VTCR
+
+ @ Use the same memory attributes for hyp. accesses as the kernel
+ @ (copy MAIRx ro HMAIRx).
+ mrc p15, 0, r0, c10, c2, 0
+ mcr p15, 4, r0, c10, c2, 0
+ mrc p15, 0, r0, c10, c2, 1
+ mcr p15, 4, r0, c10, c2, 1
+
+ @ Set the HSCTLR to:
+ @ - ARM/THUMB exceptions: ARM
+ @ - Endianness: Kernel config
+ @ - Fast Interrupt Features: Kernel config
+ @ - Write permission implies XN: disabled
+ @ - Instruction cache: enabled
+ @ - Data/Unified cache: enabled
+ @ - Memory alignment checks: enabled
+ @ - MMU: enabled (this code must be run from an identity mapping)
+ mrc p15, 4, r0, c1, c0, 0 @ HSCR
+ ldr r12, =HSCTLR_MASK
+ bic r0, r0, r12
+ mrc p15, 0, r1, c1, c0, 0 @ SCTLR
+ ldr r12, =(HSCTLR_EE | HSCTLR_FI)
+ and r1, r1, r12
+ ldr r12, =(HSCTLR_M | HSCTLR_A | HSCTLR_I)
+ orr r1, r1, r12
+ orr r0, r0, r1
+ isb
+ mcr p15, 4, r0, c1, c0, 0 @ HSCR
+ isb
+
+ @ Set stack pointer and return to the kernel
+ pop {r1, r2, r12}
+ mov sp, r1
+ eret
+
+ .ltorg
+
+ .align 12
+
+ __kvm_init_sp:
+ .globl __kvm_hyp_init_end
+__kvm_hyp_init_end:
diff --git a/arch/arm/kvm/arm_interrupts.S b/arch/arm/kvm/arm_interrupts.S
index 073a494..2edc49b 100644
--- a/arch/arm/kvm/arm_interrupts.S
+++ b/arch/arm/kvm/arm_interrupts.S
@@ -13,5 +13,35 @@
* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
+
+#include <linux/linkage.h>
+#include <asm/unified.h>
+#include <asm/page.h>
#include <asm/asm-offsets.h>
#include <asm/kvm_asm.h>
+#include <asm/kvm_arm.h>
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@ Hypervisor world-switch code
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+
+ .text
+ .arm
+
+ENTRY(__kvm_vcpu_run)
+THUMB( orr lr, lr, #1)
+ mov pc, lr
+__kvm_vcpu_run_end:
+ .globl __kvm_vcpu_run_end
+
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@ Hypervisor exception vector and handlers
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+
+ .align 5
+__kvm_hyp_vector:
+ .globl __kvm_hyp_vector
+ nop
+__kvm_hyp_vector_end:
+ .globl __kvm_hyp_vector_end
diff --git a/arch/arm/kvm/arm_mmu.c b/arch/arm/kvm/arm_mmu.c
index 2cccd48..8fefda2 100644
--- a/arch/arm/kvm/arm_mmu.c
+++ b/arch/arm/kvm/arm_mmu.c
@@ -13,3 +13,216 @@
* Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*
*/
+
+#include <linux/mman.h>
+#include <linux/kvm_host.h>
+#include <asm/pgalloc.h>
+#include <asm/kvm_arm.h>
+#include <asm/kvm_mmu.h>
+
+#include "debug.h"
+
+pgd_t *kvm_hyp_pgd;
+
+static void free_ptes(pmd_t *pmd, unsigned long addr)
+{
+ pte_t *pte;
+ unsigned int i;
+
+ for (i = 0; i < PTRS_PER_PMD; i++, addr += PMD_SIZE) {
+ if (!pmd_none(*pmd) && pmd_table(*pmd)) {
+ pte = pte_offset_kernel(pmd, addr);
+ pte_free_kernel(NULL, pte);
+ }
+ pmd++;
+ }
+}
+
+/**
+ * free_hyp_pmds - free a Hyp-mode level-2 tables and child level-3 tables
+ * @hypd_pgd: The Hyp-mode page table pointer
+ *
+ * Assumes this is a page table used strictly in Hyp-mode and therefore contains
+ * only mappings in the kernel memory area, which is above PAGE_OFFSET.
+ */
+void free_hyp_pmds(pgd_t *hyp_pgd)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ unsigned long addr, next, end;
+
+ addr = PAGE_OFFSET;
+ end = ~0;
+ do {
+ next = pgd_addr_end(addr, end);
+ pgd = hyp_pgd + pgd_index(addr);
+ pud = pud_offset(pgd, addr);
+
+ BUG_ON(pud_bad(*pud));
+
+ if (pud_none(*pud))
+ continue;
+
+ pmd = pmd_offset(pud, addr);
+ free_ptes(pmd, addr);
+ pmd_free(NULL, pmd);
+ } while (addr = next, addr != end);
+}
+
+static void remove_hyp_pte_mappings(pmd_t *pmd, unsigned long addr,
+ unsigned long end)
+{
+ pte_t *pte;
+
+ do {
+ pte = pte_offset_kernel(pmd, addr);
+ pte_clear(NULL, addr, pte);
+ } while (addr += PAGE_SIZE, addr < end);
+}
+
+static void remove_hyp_pmd_mappings(pgd_t *pgd, unsigned long addr,
+ unsigned long end)
+{
+ pud_t *pud;
+ pmd_t *pmd;
+ unsigned long next;
+
+ do {
+ next = pmd_addr_end(addr, end);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
+
+ BUG_ON(pmd_sect(*pmd));
+
+ if (!pmd_none(*pmd))
+ remove_hyp_pte_mappings(pmd, addr, next);
+ } while (addr = next, addr < end);
+}
+
+/**
+ * remove_hyp_mappings - clear hypervisor mappings from specified range
+ * @hyp_pgd: The Hyp-mode page table pointer
+ * @start: The start virtual address of the area to clear
+ * @end: The end virtual address of the area to clear
+ *
+ * The page tables aren't actually freed - call free_hyp_pmds to do this.
+ */
+void remove_hyp_mappings(pgd_t *hyp_pgd, unsigned long start,
+ unsigned long end)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ unsigned long addr, next;
+
+ BUG_ON(start > end);
+ BUG_ON(start < PAGE_OFFSET);
+
+ addr = start;
+ do {
+ next = pgd_addr_end(addr, end);
+ pgd = hyp_pgd + pgd_index(addr);
+ pud = pud_offset(pgd, addr);
+
+ BUG_ON(pud_bad(*pud));
+
+ if (pud_none(*pud))
+ continue;
+
+ remove_hyp_pmd_mappings(pgd, addr, next);
+ } while (addr = next, addr < end);
+}
+
+static void create_hyp_pte_mappings(pmd_t *pmd, unsigned long addr,
+ unsigned long end)
+{
+ pte_t *pte;
+ struct page *page;
+
+ addr &= PAGE_MASK;
+ do {
+ pte = pte_offset_kernel(pmd, addr);
+ BUG_ON(!virt_addr_valid(addr));
+ page = virt_to_page(addr);
+
+ set_pte_ext(pte, mk_pte(page, PAGE_HYP), 0);
+ } while (addr += PAGE_SIZE, addr < end);
+}
+
+static int create_hyp_pmd_mappings(pud_t *pud, unsigned long addr,
+ unsigned long end)
+{
+ pmd_t *pmd;
+ pte_t *pte;
+ unsigned long next;
+
+ do {
+ next = pmd_addr_end(addr, end);
+ pmd = pmd_offset(pud, addr);
+
+ BUG_ON(pmd_sect(*pmd));
+
+ if (pmd_none(*pmd)) {
+ pte = pte_alloc_one_kernel(NULL, addr);
+ if (!pte) {
+ kvm_err(-ENOMEM, "Cannot allocate Hyp pte");
+ return -ENOMEM;
+ }
+ pmd_populate_kernel(NULL, pmd, pte);
+ }
+
+ create_hyp_pte_mappings(pmd, addr, next);
+ } while (addr = next, addr < end);
+
+ return 0;
+}
+
+/**
+ * create_hyp_mappings - map a kernel virtual address range in Hyp mode
+ * @hyp_pgd: The allocated hypervisor level-1 table
+ * @start: The virtual kernel start address of the range
+ * @end: The virtual kernel end address of the range
+ *
+ * The same virtual address as the kernel virtual address is also used in
+ * Hyp-mode mapping to the same underlying physical pages.
+ */
+int create_hyp_mappings(pgd_t *hyp_pgd, unsigned long start, unsigned long end)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ unsigned long addr, next;
+ int err = 0;
+
+ BUG_ON(start > end);
+ if (start < PAGE_OFFSET)
+ return -EINVAL;
+
+ addr = start;
+ do {
+ next = pgd_addr_end(addr, end);
+ pgd = hyp_pgd + pgd_index(addr);
+ pud = pud_offset(pgd, addr);
+
+ if (pud_none_or_clear_bad(pud)) {
+ pmd = pmd_alloc_one(NULL, addr);
+ if (!pmd) {
+ kvm_err(-ENOMEM, "Cannot allocate Hyp pmd");
+ return -ENOMEM;
+ }
+ pud_populate(NULL, pud, pmd);
+ }
+
+ err = create_hyp_pmd_mappings(pud, addr, next);
+ if (err)
+ return err;
+ } while (addr = next, addr < end);
+
+ return err;
+}
+
+int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+ KVMARM_NOT_IMPLEMENTED();
+ return -EINVAL;
+}
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 83dc20d..37b8b4c 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -1,5 +1,6 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/module.h>
#include <linux/kernel.h>
#include <asm/cputype.h>
@@ -121,12 +122,15 @@ void hyp_identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
{
__identity_mapping_add(pgd, addr, end, true);
}
+EXPORT_SYMBOL_GPL(hyp_identity_mapping_add);
static void hyp_idmap_del_pmd(pgd_t *pgd, unsigned long addr)
{
+ pud_t *pud;
pmd_t *pmd;
- pmd = pmd_offset(pgd, addr);
+ pud = pud_offset(pgd, addr);
+ pmd = pmd_offset(pud, addr);
pmd_free(NULL, pmd);
}
@@ -148,6 +152,7 @@ void hyp_identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
}
} while (addr = next, addr < end);
}
+EXPORT_SYMBOL_GPL(hyp_identity_mapping_del);
#endif
/*
diff --git a/mm/memory.c b/mm/memory.c
index 61e66f0..2a99b7b 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -204,6 +204,7 @@ void pgd_clear_bad(pgd_t *pgd)
pgd_ERROR(*pgd);
pgd_clear(pgd);
}
+EXPORT_SYMBOL_GPL(pgd_clear_bad);
void pud_clear_bad(pud_t *pud)
{
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
