This stores the MMU mode (real vs. virt) in the PACA on exceptions
by passing bit 0x4000 to the KVM trampoline. This unfortunately
forces us to add a few more trampolines due to how our macros
work.
Signed-off-by: Benjamin Herrenschmidt <benh@xxxxxxxxxxxxxxxxxxx>
---
arch/powerpc/include/asm/exception-64s.h | 35 +++++++++++++++----------
arch/powerpc/include/asm/head-64.h | 8 ++++++
arch/powerpc/include/asm/kvm_book3s_asm.h | 1 +
arch/powerpc/kernel/asm-offsets.c | 1 +
arch/powerpc/kernel/exceptions-64s.S | 43 +++++++++++++++++++++++++++----
arch/powerpc/kvm/book3s_hv_rmhandlers.S | 9 +++++--
6 files changed, 76 insertions(+), 21 deletions(-)
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index b27205297e1d..64936a89c79c 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -243,10 +243,10 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
EXCEPTION_PROLOG_1(area, extra, vec); \
EXCEPTION_PROLOG_PSERIES_1(label, h);
-#define __KVMTEST(h, n) \
+#define __KVMTEST(h, n, v) \
lbz r10,HSTATE_IN_GUEST(r13); \
cmpwi r10,0; \
- bne do_kvm_##h##n
+ bne do_kvm_##h##n##v
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/*
@@ -353,12 +353,12 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
b kvmppc_skip_##h##interrupt
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
-#define KVMTEST(h, n) __KVMTEST(h, n)
+#define KVMTEST(h, n, v) __KVMTEST(h, n, v)
#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n)
#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n)
#else
-#define KVMTEST(h, n)
+#define KVMTEST(h, n, v)
#define KVM_HANDLER(area, h, n)
#define KVM_HANDLER_SKIP(area, h, n)
#endif
@@ -479,13 +479,13 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \
EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_STD)
-#define STD_RELON_EXCEPTION_HV(loc, vec, label) \
- SET_SCRATCH0(r13); /* save r13 */ \
+#define STD_RELON_EXCEPTION_HV(loc, vec, label) \
+ SET_SCRATCH0(r13); /* save r13 */ \
EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, \
- EXC_HV, KVMTEST_HV, vec);
+ EXC_HV, KVMTEST_HV_VIRT, vec);
#define STD_RELON_EXCEPTION_HV_OOL(vec, label) \
- EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \
+ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV_VIRT, vec); \
EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
/* This associate vector numbers with bits in paca->irq_happened */
@@ -506,18 +506,25 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
#define _SOFTEN_TEST(h, vec) __SOFTEN_TEST(h, vec)
#define SOFTEN_TEST_PR(vec) \
- KVMTEST(EXC_STD, vec); \
+ KVMTEST(EXC_STD, vec,); \
_SOFTEN_TEST(EXC_STD, vec)
#define SOFTEN_TEST_HV(vec) \
- KVMTEST(EXC_HV, vec); \
+ KVMTEST(EXC_HV, vec,); \
+ _SOFTEN_TEST(EXC_HV, vec)
+
+#define SOFTEN_TEST_HV_VIRT(vec) \
+ KVMTEST(EXC_HV, vec, _VIRT); \
_SOFTEN_TEST(EXC_HV, vec)
#define KVMTEST_PR(vec) \
- KVMTEST(EXC_STD, vec)
+ KVMTEST(EXC_STD, vec,)
#define KVMTEST_HV(vec) \
- KVMTEST(EXC_HV, vec)
+ KVMTEST(EXC_HV, vec,)
+
+#define KVMTEST_HV_VIRT(vec) \
+ KVMTEST(EXC_HV, vec, _VIRT)
#define SOFTEN_NOTEST_PR(vec) _SOFTEN_TEST(EXC_STD, vec)
#define SOFTEN_NOTEST_HV(vec) _SOFTEN_TEST(EXC_HV, vec)
@@ -562,10 +569,10 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label) \
_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \
- EXC_HV, SOFTEN_TEST_HV)
+ EXC_HV, SOFTEN_TEST_HV_VIRT)
#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label) \
- EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \
+ EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV_VIRT, vec); \
EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
/*
diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h
index 0a663dfc28b5..faf8d03f0797 100644
--- a/arch/powerpc/include/asm/head-64.h
+++ b/arch/powerpc/include/asm/head-64.h
@@ -404,6 +404,14 @@ end_##sname:
TRAMP_KVM_BEGIN(do_kvm_H##n); \
KVM_HANDLER_SKIP(area, EXC_HV, n + 0x2); \
+#define TRAMP_KVM_HV_VIRT(area, n) \
+ TRAMP_KVM_BEGIN(do_kvm_H##n##_VIRT); \
+ KVM_HANDLER(area, EXC_HV, n + 0x4002); \
+
+#define TRAMP_KVM_HV_VIRT_SKIP(area, n) \
+ TRAMP_KVM_BEGIN(do_kvm_H##n##_VIRT); \
+ KVM_HANDLER_SKIP(area, EXC_HV, n + 0x4002); \
+
#define EXC_COMMON(name, realvec, hdlr) \
EXC_COMMON_BEGIN(name); \
STD_EXCEPTION_COMMON(realvec, name, hdlr); \
diff --git a/arch/powerpc/include/asm/kvm_book3s_asm.h b/arch/powerpc/include/asm/kvm_book3s_asm.h
index ab386af2904f..c6afd99d91fb 100644
--- a/arch/powerpc/include/asm/kvm_book3s_asm.h
+++ b/arch/powerpc/include/asm/kvm_book3s_asm.h
@@ -125,6 +125,7 @@ struct kvmppc_host_state {
void __iomem *xive_tima_phys;
void __iomem *xive_tima_virt;
u32 saved_xirr;
+ u32 exit_virt;
u64 dabr;
u64 host_mmcr[7]; /* MMCR 0,1,A, SIAR, SDAR, MMCR2, SIER */
u32 host_pmc[8];
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 1672dffd94e2..25ee619531c2 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -641,6 +641,7 @@ int main(void)
HSTATE_FIELD(HSTATE_XIVE_TIMA_PHYS, xive_tima_phys);
HSTATE_FIELD(HSTATE_XIVE_TIMA_VIRT, xive_tima_virt);
HSTATE_FIELD(HSTATE_SAVED_XIRR, saved_xirr);
+ HSTATE_FIELD(HSTATE_EXIT_VIRT, exit_virt);
HSTATE_FIELD(HSTATE_HOST_IPI, host_ipi);
HSTATE_FIELD(HSTATE_PTID, ptid);
HSTATE_FIELD(HSTATE_TID, tid);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 175891c6909c..845426266e9a 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -702,7 +702,7 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
.globl hardware_interrupt_relon_hv;
hardware_interrupt_relon_hv:
BEGIN_FTR_SECTION
- _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_HV, SOFTEN_TEST_HV)
+ _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_HV, SOFTEN_TEST_HV_VIRT)
FTR_SECTION_ELSE
_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_STD, SOFTEN_TEST_PR)
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
@@ -710,6 +710,7 @@ EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
TRAMP_KVM(PACA_EXGEN, 0x500)
TRAMP_KVM_HV(PACA_EXGEN, 0x500)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0x500)
EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ)
@@ -809,6 +810,7 @@ EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
EXC_REAL_HV(hdecrementer, 0x980, 0x80)
EXC_VIRT_HV(hdecrementer, 0x4980, 0x80, 0x980)
TRAMP_KVM_HV(PACA_EXGEN, 0x980)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0x980)
EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)
@@ -872,21 +874,31 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception)
* Userspace syscalls have already saved the PPR, hcalls must save
* it before setting HMT_MEDIUM.
*/
-#define SYSCALL_KVMTEST \
+#define __SYSCALL_KVMTEST(test) \
mtctr r13; \
GET_PACA(r13); \
std r10,PACA_EXGEN+EX_R10(r13); \
- KVMTEST_PR(0xc00); /* uses r10, branch to do_kvm_0xc00_system_call */ \
+ test;/* uses r10, branch to do_kvm_0xc00_system_call */ \
HMT_MEDIUM; \
mfctr r9;
+/* 0xc00 is a normal real mode "PR" (uses STD SRR0/1) entry */
+#define SYSCALL_KVMTEST __SYSCALL_KVMTEST(KVMTEST_PR(0xc00))
+
+/* 0x4c00 is special, it can come from the guest using SRR0/1 in virtual
+ * mode. It's the only to do that and only with HV KVM so we open code
+ * the specific variant of KVMTEST macro for it
+ */
+#define SYSCALL_KVMTEST_RELON __SYSCALL_KVMTEST(KVMTEST(EXC_STD,0xc00,_VIRT))
+
#else
#define SYSCALL_KVMTEST \
HMT_MEDIUM; \
mr r9,r13; \
GET_PACA(r13);
+#define SYSCALL_KVMTEST_RELON SYSCALL_KVMTEST
#endif
-
+
#define LOAD_SYSCALL_HANDLER(reg) \
__LOAD_HANDLER(reg, system_call_common)
@@ -955,7 +967,7 @@ EXC_REAL_BEGIN(system_call, 0xc00, 0x100)
EXC_REAL_END(system_call, 0xc00, 0x100)
EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
- SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */
+ SYSCALL_KVMTEST_RELON /* loads PACA into r13, and saves r13 to r9 */
SYSCALL_FASTENDIAN_TEST
SYSCALL_VIRT
SYSCALL_FASTENDIAN
@@ -983,6 +995,21 @@ TRAMP_KVM_BEGIN(do_kvm_0xc00)
std r9,PACA_EXGEN+EX_R9(r13)
mfcr r9
KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
+
+TRAMP_KVM_BEGIN(do_kvm_0xc00_VIRT)
+ /*
+ * Save the PPR (on systems that support it) before changing to
+ * HMT_MEDIUM. That allows the KVM code to save that value into the
+ * guest state (it is the guest's PPR value).
+ */
+ OPT_GET_SPR(r10, SPRN_PPR, CPU_FTR_HAS_PPR)
+ HMT_MEDIUM
+ OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r10, CPU_FTR_HAS_PPR)
+ mfctr r10
+ SET_SCRATCH0(r10)
+ std r9,PACA_EXGEN+EX_R9(r13)
+ mfcr r9
+ KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x4c00)
#endif
@@ -994,6 +1021,7 @@ EXC_COMMON(single_step_common, 0xd00, single_step_exception)
EXC_REAL_OOL_HV(h_data_storage, 0xe00, 0x20)
EXC_VIRT_OOL_HV(h_data_storage, 0x4e00, 0x20, 0xe00)
TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0xe00)
+TRAMP_KVM_HV_VIRT_SKIP(PACA_EXGEN, 0xe00)
EXC_COMMON_BEGIN(h_data_storage_common)
mfspr r10,SPRN_HDAR
std r10,PACA_EXGEN+EX_DAR(r13)
@@ -1010,12 +1038,14 @@ EXC_COMMON_BEGIN(h_data_storage_common)
EXC_REAL_OOL_HV(h_instr_storage, 0xe20, 0x20)
EXC_VIRT_OOL_HV(h_instr_storage, 0x4e20, 0x20, 0xe20)
TRAMP_KVM_HV(PACA_EXGEN, 0xe20)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0xe20)
EXC_COMMON(h_instr_storage_common, 0xe20, unknown_exception)
EXC_REAL_OOL_HV(emulation_assist, 0xe40, 0x20)
EXC_VIRT_OOL_HV(emulation_assist, 0x4e40, 0x20, 0xe40)
TRAMP_KVM_HV(PACA_EXGEN, 0xe40)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0xe40)
EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt)
@@ -1086,6 +1116,7 @@ EXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, handle_hmi_exception,
EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20)
EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80)
TRAMP_KVM_HV(PACA_EXGEN, 0xe80)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0xe80)
#ifdef CONFIG_PPC_DOORBELL
EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception)
#else
@@ -1096,6 +1127,7 @@ EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception)
EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20)
EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0)
TRAMP_KVM_HV(PACA_EXGEN, 0xea0)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0xea0)
EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ)
@@ -1193,6 +1225,7 @@ EXC_COMMON(facility_unavailable_common, 0xf60, facility_unavailable_exception)
EXC_REAL_OOL_HV(h_facility_unavailable, 0xf80, 0x20)
EXC_VIRT_OOL_HV(h_facility_unavailable, 0x4f80, 0x20, 0xf80)
TRAMP_KVM_HV(PACA_EXGEN, 0xf80)
+TRAMP_KVM_HV_VIRT(PACA_EXGEN, 0xf80)
EXC_COMMON(h_facility_unavailable_common, 0xf80, facility_unavailable_exception)
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index a38df400c1b7..1e32e188ba17 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -95,6 +95,8 @@ BEGIN_FTR_SECTION
46:
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
+ li r0,0
+ stw r0,HSTATE_EXIT_VIRT(r13)
ld r4, HSTATE_KVM_VCPU(r13)
bl kvmppc_hv_entry
@@ -1312,8 +1314,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
mfspr r11, SPRN_SRR1
std r10, VCPU_SRR0(r9)
std r11, VCPU_SRR1(r9)
- /* trap is in the low half of r12, clear CR from the high half */
- clrldi r12, r12, 32
+ /* Recover bit 0x4000 from trap (virt vs real mode) */
+ rlwinm r0,r12,18,31,31
+ stw r0,HSTATE_EXIT_VIRT(r13)
+ /* trap is in the low half of r12, clear CR in high half and virt mode bit */
+ clrldi r12, r12, 50
andi. r0, r12, 2 /* need to read HSRR0/1? */
beq 1f
mfspr r10, SPRN_HSRR0
--
2.14.3
