These sets of state are moved together as nVHE's speculative AT
workaround depends on their correct interaction. As a consequence of
this change, the workaround is much simpler as both the host and the
guests now share the same code path.
Signed-off-by: Andrew Scull <ascull@xxxxxxxxxx>
---
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h | 48 ++++-------
arch/arm64/kvm/hyp/nvhe/switch.c | 96 +++++++++-------------
2 files changed, 54 insertions(+), 90 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
index 0c24c922bae8..cffe7dd3df41 100644
--- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
+++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
@@ -75,25 +75,6 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
{
write_sysreg(ctxt_sys_reg(ctxt, MPIDR_EL1), vmpidr_el2);
write_sysreg(ctxt_sys_reg(ctxt, CSSELR_EL1), csselr_el1);
-
- if (has_vhe() ||
- !cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
- write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
- write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR);
- } else if (!ctxt->is_host) {
- /*
- * Must only be done for guest registers, hence the context
- * test. Pairs with nVHE's __activate_traps().
- */
- write_sysreg_el1((ctxt_sys_reg(ctxt, TCR_EL1) |
- TCR_EPD1_MASK | TCR_EPD0_MASK),
- SYS_TCR);
- isb();
- write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1] | SCTLR_ELx_M,
- SYS_SCTLR);
- isb();
- }
-
write_sysreg_el1(ctxt_sys_reg(ctxt, CPACR_EL1), SYS_CPACR);
write_sysreg_el1(ctxt_sys_reg(ctxt, TTBR0_EL1), SYS_TTBR0);
write_sysreg_el1(ctxt_sys_reg(ctxt, TTBR1_EL1), SYS_TTBR1);
@@ -109,23 +90,24 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
write_sysreg(ctxt_sys_reg(ctxt, PAR_EL1), par_el1);
write_sysreg(ctxt_sys_reg(ctxt, TPIDR_EL1), tpidr_el1);
- if (!has_vhe() &&
- cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT) &&
- ctxt->is_host) {
+ if (!has_vhe()) {
/*
- * Must only be done for host registers, hence the context
- * test. Pairs with nVHE's __deactivate_traps().
+ * For CPUs that are affected by the speculative AT errata,
+ * ensure the vcpu's stage 1 and stage 2 translations have been
+ * configured before before updating TCR_EL1 and SCTLR_EL1 to
+ * potentially allow any speculative walks to occur. The stage
+ * 2 will have already been configured by the nVHE switching
+ * code before calling this function.
*/
- isb();
- /*
- * At this stage, and thanks to the above isb(), S2 is
- * deconfigured and disabled. We can now restore the host's
- * S1 configuration: SCTLR, and only then TCR.
- */
- write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
- isb();
- write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR);
+ asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
+ }
+
+ write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
+ if (!has_vhe()) {
+ /* Ensure S1 MMU state is restored before allowing S1 walks. */
+ asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
}
+ write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR);
write_sysreg(ctxt_sys_reg(ctxt, SP_EL1), sp_el1);
write_sysreg_el1(ctxt_sys_reg(ctxt, ELR_EL1), SYS_ELR);
diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 25c64392356b..c87b0a709d35 100644
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -43,43 +43,12 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
}
write_sysreg(val, cptr_el2);
-
- if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
- struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
-
- isb();
- /*
- * At this stage, and thanks to the above isb(), S2 is
- * configured and enabled. We can now restore the guest's S1
- * configuration: SCTLR, and only then TCR.
- */
- write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR_EL1), SYS_SCTLR);
- isb();
- write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1), SYS_TCR);
- }
}
static void __deactivate_traps(struct kvm_vcpu *host_vcpu, struct kvm_vcpu *vcpu)
{
___deactivate_traps(vcpu);
- if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
- u64 val;
-
- /*
- * Set the TCR and SCTLR registers in the exact opposite
- * sequence as __activate_traps (first prevent walks,
- * then force the MMU on). A generous sprinkling of isb()
- * ensure that things happen in this exact order.
- */
- val = read_sysreg_el1(SYS_TCR);
- write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR);
- isb();
- val = read_sysreg_el1(SYS_SCTLR);
- write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR);
- isb();
- }
-
__deactivate_traps_common();
write_sysreg(host_vcpu->arch.mdcr_el2, mdcr_el2);
@@ -87,9 +56,12 @@ static void __deactivate_traps(struct kvm_vcpu *host_vcpu, struct kvm_vcpu *vcpu
write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
}
-static void __deactivate_vm(struct kvm_vcpu *vcpu)
+static void __restore_stage2(struct kvm_vcpu *vcpu)
{
- write_sysreg(0, vttbr_el2);
+ if (vcpu->arch.hcr_el2 & HCR_VM)
+ __activate_vm(kern_hyp_va(vcpu->arch.hw_mmu));
+ else
+ write_sysreg(0, vttbr_el2);
}
/* Save VGICv3 state on non-VHE systems */
@@ -147,8 +119,6 @@ static void __pmu_switch_to_host(void)
static void __kvm_vcpu_switch_to_guest(struct kvm_vcpu *host_vcpu,
struct kvm_vcpu *vcpu)
{
- struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
-
/*
* Having IRQs masked via PMR when entering the guest means the GIC
* will not signal the CPU of interrupts of lower priority, and the
@@ -162,35 +132,14 @@ static void __kvm_vcpu_switch_to_guest(struct kvm_vcpu *host_vcpu,
__pmu_switch_to_guest();
- /*
- * We must restore the 32-bit state before the sysregs, thanks
- * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
- *
- * Also, and in order to be able to deal with erratum #1319537 (A57)
- * and #1319367 (A72), we must ensure that all VM-related sysreg are
- * restored before we enable S2 translation.
- */
- __sysreg32_restore_state(vcpu);
- __sysreg_restore_state_nvhe(guest_ctxt);
-
- __activate_vm(kern_hyp_va(vcpu->arch.hw_mmu));
- __activate_traps(vcpu);
-
__timer_enable_traps(vcpu);
}
static void __kvm_vcpu_switch_to_host(struct kvm_vcpu *host_vcpu,
struct kvm_vcpu *vcpu)
{
- struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
-
__timer_disable_traps(vcpu);
- __deactivate_traps(host_vcpu, vcpu);
- __deactivate_vm(vcpu);
-
- __sysreg_restore_state_nvhe(&host_vcpu->arch.ctxt);
-
__pmu_switch_to_host();
/* Returning to host will clear PSR.I, remask PMR if needed */
@@ -228,6 +177,39 @@ static void __vcpu_restore_state(struct kvm_vcpu *vcpu, bool restore_debug)
else
__kvm_vcpu_switch_to_guest(running_vcpu, vcpu);
+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
+ u64 val;
+
+ /*
+ * Set the TCR and SCTLR registers to prevent any stage 1 or 2
+ * page table walks or TLB allocations. A generous sprinkling
+ * of isb() ensure that things happen in this exact order.
+ */
+ val = read_sysreg_el1(SYS_TCR);
+ write_sysreg_el1(val | TCR_EPD1_MASK | TCR_EPD0_MASK, SYS_TCR);
+ isb();
+ val = read_sysreg_el1(SYS_SCTLR);
+ write_sysreg_el1(val | SCTLR_ELx_M, SYS_SCTLR);
+ isb();
+ }
+
+ /*
+ * We must restore the 32-bit state before the sysregs, thanks to
+ * erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
+ *
+ * Also, and in order to deal with the speculative AT errata, we must
+ * ensure the S2 translation is restored before allowing page table
+ * walks and TLB allocations when the sysregs are restored.
+ */
+ __restore_stage2(vcpu);
+ __sysreg32_restore_state(vcpu);
+ __sysreg_restore_state_nvhe(&vcpu->arch.ctxt);
+
+ if (vcpu->arch.ctxt.is_host)
+ __deactivate_traps(vcpu, running_vcpu);
+ else
+ __activate_traps(vcpu);
+
__hyp_vgic_restore_state(vcpu);
/*
@@ -300,7 +282,7 @@ void __noreturn hyp_panic(void)
if (vcpu != host_vcpu) {
__timer_disable_traps(vcpu);
__deactivate_traps(host_vcpu, vcpu);
- __deactivate_vm(vcpu);
+ __restore_stage2(host_vcpu);
__sysreg_restore_state_nvhe(&host_vcpu->arch.ctxt);
}
--
2.27.0.389.gc38d7665816-goog
_______________________________________________
kvmarm mailing list
kvmarm@xxxxxxxxxxxxxxxxxxxxx
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
