Implement loongarch vcpu world switch, including vcpu enter guest and vcpu exit from guest, both operations need to save or restore the host and guest registers. Signed-off-by: Tianrui Zhao <zhaotianrui@xxxxxxxxxxx> --- arch/loongarch/kernel/asm-offsets.c | 32 +++ arch/loongarch/kvm/switch.S | 327 ++++++++++++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 arch/loongarch/kvm/switch.S diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 4bdb203fc..655741c03 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -9,6 +9,7 @@ #include <linux/mm.h> #include <linux/kbuild.h> #include <linux/suspend.h> +#include <linux/kvm_host.h> #include <asm/cpu-info.h> #include <asm/ptrace.h> #include <asm/processor.h> @@ -272,3 +273,34 @@ void output_pbe_defines(void) BLANK(); } #endif + +void output_kvm_defines(void) +{ + COMMENT(" KVM/LOONGARCH Specific offsets. "); + + OFFSET(VCPU_FCSR0, kvm_vcpu_arch, fpu.fcsr); + OFFSET(VCPU_FCC, kvm_vcpu_arch, fpu.fcc); + BLANK(); + + OFFSET(KVM_VCPU_ARCH, kvm_vcpu, arch); + OFFSET(KVM_VCPU_KVM, kvm_vcpu, kvm); + OFFSET(KVM_VCPU_RUN, kvm_vcpu, run); + BLANK(); + + OFFSET(KVM_ARCH_HSTACK, kvm_vcpu_arch, host_stack); + OFFSET(KVM_ARCH_HGP, kvm_vcpu_arch, host_gp); + OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); + OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); + OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry); + OFFSET(KVM_ARCH_GPC, kvm_vcpu_arch, pc); + OFFSET(KVM_ARCH_GGPR, kvm_vcpu_arch, gprs); + OFFSET(KVM_ARCH_HESTAT, kvm_vcpu_arch, host_estat); + OFFSET(KVM_ARCH_HBADV, kvm_vcpu_arch, badv); + OFFSET(KVM_ARCH_HBADI, kvm_vcpu_arch, badi); + OFFSET(KVM_ARCH_HECFG, kvm_vcpu_arch, host_ecfg); + OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); + OFFSET(KVM_ARCH_HPERCPU, kvm_vcpu_arch, host_percpu); + + OFFSET(KVM_GPGD, kvm, arch.gpa_mm.pgd); + BLANK(); +} diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S new file mode 100644 index 000000000..c0b8062ac --- /dev/null +++ b/arch/loongarch/kvm/switch.S @@ -0,0 +1,327 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2023 Loongson Technology Corporation Limited + */ + +#include <linux/linkage.h> +#include <asm/stackframe.h> +#include <asm/asm.h> +#include <asm/asmmacro.h> +#include <asm/regdef.h> +#include <asm/loongarch.h> +#include <asm/export.h> + +#define RESUME_HOST (1 << 1) + +#define PT_GPR_OFFSET(x) (PT_R0 + 8*x) +#define CONFIG_GUEST_CRMD ((1 << CSR_CRMD_DACM_SHIFT) | \ + (1 << CSR_CRMD_DACF_SHIFT) | \ + CSR_CRMD_PG | PLV_KERN) + .text + +.macro kvm_save_host_gpr base + .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 + st.d $r\n, \base, PT_GPR_OFFSET(\n) + .endr +.endm + +.macro kvm_restore_host_gpr base + .irp n,1,2,3,22,23,24,25,26,27,28,29,30,31 + ld.d $r\n, \base, PT_GPR_OFFSET(\n) + .endr +.endm + +/* + * prepare switch to guest + * @param: + * KVM_ARCH: kvm_vcpu_arch, don't touch it until 'ertn' + * GPRNUM: KVM_ARCH gpr number + * tmp, tmp1: temp register + */ +.macro kvm_switch_to_guest KVM_ARCH GPRNUM tmp tmp1 + /* set host excfg.VS=0, all exceptions share one exception entry */ + csrrd \tmp, LOONGARCH_CSR_ECFG + bstrins.w \tmp, zero, CSR_ECFG_VS_SHIFT_END, CSR_ECFG_VS_SHIFT + csrwr \tmp, LOONGARCH_CSR_ECFG + + /* Load up the new EENTRY */ + ld.d \tmp, \KVM_ARCH, KVM_ARCH_GEENTRY + csrwr \tmp, LOONGARCH_CSR_EENTRY + + /* Set Guest ERA */ + ld.d \tmp, \KVM_ARCH, KVM_ARCH_GPC + csrwr \tmp, LOONGARCH_CSR_ERA + + /* Save host PGDL */ + csrrd \tmp, LOONGARCH_CSR_PGDL + st.d \tmp, \KVM_ARCH, KVM_ARCH_HPGD + + /* Switch to kvm */ + ld.d \tmp1, \KVM_ARCH, KVM_VCPU_KVM - KVM_VCPU_ARCH + + /* Load guest PGDL */ + lu12i.w \tmp, KVM_GPGD + srli.w \tmp, \tmp, 12 + ldx.d \tmp, \tmp1, \tmp + csrwr \tmp, LOONGARCH_CSR_PGDL + + /* Mix GID and RID */ + csrrd \tmp1, LOONGARCH_CSR_GSTAT + bstrpick.w \tmp1, \tmp1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT + csrrd \tmp, LOONGARCH_CSR_GTLBC + bstrins.w \tmp, \tmp1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT + csrwr \tmp, LOONGARCH_CSR_GTLBC + + /* + * Switch to guest: + * GSTAT.PGM = 1, ERRCTL.ISERR = 0, TLBRPRMD.ISTLBR = 0 + * ertn + */ + + /* Prepare enable Intr before enter guest */ + ori \tmp, zero, CSR_PRMD_PIE + csrxchg \tmp, \tmp, LOONGARCH_CSR_PRMD + + /* Set PVM bit to setup ertn to guest context */ + ori \tmp, zero, CSR_GSTAT_PVM + csrxchg \tmp, \tmp, LOONGARCH_CSR_GSTAT + + /* Load Guest gprs */ + ld.d $r1, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 1) + ld.d $r2, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 2) + ld.d $r3, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 3) + ld.d $r4, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 4) + ld.d $r5, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 5) + ld.d $r7, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 7) + ld.d $r8, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 8) + ld.d $r9, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 9) + ld.d $r10, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 10) + ld.d $r11, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 11) + ld.d $r12, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 12) + ld.d $r13, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 13) + ld.d $r14, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 14) + ld.d $r15, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 15) + ld.d $r16, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 16) + ld.d $r17, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 17) + ld.d $r18, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 18) + ld.d $r19, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 19) + ld.d $r20, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 20) + ld.d $r21, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 21) + ld.d $r22, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 22) + ld.d $r23, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 23) + ld.d $r24, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 24) + ld.d $r25, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 25) + ld.d $r26, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 26) + ld.d $r27, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 27) + ld.d $r28, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 28) + ld.d $r29, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 29) + ld.d $r30, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 30) + ld.d $r31, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * 31) + /* Load KVM_ARCH register */ + ld.d \KVM_ARCH, \KVM_ARCH, (KVM_ARCH_GGPR + 8 * \GPRNUM) + + ertn +.endm + +/* load kvm_vcpu to a2 and store a1 for free use */ + .section .text + .cfi_sections .debug_frame +SYM_CODE_START(kvm_vector_entry) + csrwr a2, KVM_TEMP_KS + csrrd a2, KVM_VCPU_KS + addi.d a2, a2, KVM_VCPU_ARCH + + /* After save gprs, free to use any gpr */ + st.d $r1, a2, (KVM_ARCH_GGPR + 8 * 1) + st.d $r2, a2, (KVM_ARCH_GGPR + 8 * 2) + st.d $r3, a2, (KVM_ARCH_GGPR + 8 * 3) + st.d $r4, a2, (KVM_ARCH_GGPR + 8 * 4) + st.d $r5, a2, (KVM_ARCH_GGPR + 8 * 5) + st.d $r7, a2, (KVM_ARCH_GGPR + 8 * 7) + st.d $r8, a2, (KVM_ARCH_GGPR + 8 * 8) + st.d $r9, a2, (KVM_ARCH_GGPR + 8 * 9) + st.d $r10, a2, (KVM_ARCH_GGPR + 8 * 10) + st.d $r11, a2, (KVM_ARCH_GGPR + 8 * 11) + st.d $r12, a2, (KVM_ARCH_GGPR + 8 * 12) + st.d $r13, a2, (KVM_ARCH_GGPR + 8 * 13) + st.d $r14, a2, (KVM_ARCH_GGPR + 8 * 14) + st.d $r15, a2, (KVM_ARCH_GGPR + 8 * 15) + st.d $r16, a2, (KVM_ARCH_GGPR + 8 * 16) + st.d $r17, a2, (KVM_ARCH_GGPR + 8 * 17) + st.d $r18, a2, (KVM_ARCH_GGPR + 8 * 18) + st.d $r19, a2, (KVM_ARCH_GGPR + 8 * 19) + st.d $r20, a2, (KVM_ARCH_GGPR + 8 * 20) + st.d $r21, a2, (KVM_ARCH_GGPR + 8 * 21) + st.d $r22, a2, (KVM_ARCH_GGPR + 8 * 22) + st.d $r23, a2, (KVM_ARCH_GGPR + 8 * 23) + st.d $r24, a2, (KVM_ARCH_GGPR + 8 * 24) + st.d $r25, a2, (KVM_ARCH_GGPR + 8 * 25) + st.d $r26, a2, (KVM_ARCH_GGPR + 8 * 26) + st.d $r27, a2, (KVM_ARCH_GGPR + 8 * 27) + st.d $r28, a2, (KVM_ARCH_GGPR + 8 * 28) + st.d $r29, a2, (KVM_ARCH_GGPR + 8 * 29) + st.d $r30, a2, (KVM_ARCH_GGPR + 8 * 30) + st.d $r31, a2, (KVM_ARCH_GGPR + 8 * 31) + /* Save guest a2 */ + csrrd t0, KVM_TEMP_KS + st.d t0, a2, (KVM_ARCH_GGPR + 8 * REG_A2) + + /* a2: kvm_vcpu_arch, a1 is free to use */ + csrrd s1, KVM_VCPU_KS + ld.d s0, s1, KVM_VCPU_RUN + + csrrd t0, LOONGARCH_CSR_ESTAT + st.d t0, a2, KVM_ARCH_HESTAT + csrrd t0, LOONGARCH_CSR_ERA + st.d t0, a2, KVM_ARCH_GPC + csrrd t0, LOONGARCH_CSR_BADV + st.d t0, a2, KVM_ARCH_HBADV + csrrd t0, LOONGARCH_CSR_BADI + st.d t0, a2, KVM_ARCH_HBADI + + /* Restore host excfg.VS */ + csrrd t0, LOONGARCH_CSR_ECFG + ld.d t1, a2, KVM_ARCH_HECFG + or t0, t0, t1 + csrwr t0, LOONGARCH_CSR_ECFG + + /* Restore host eentry */ + ld.d t0, a2, KVM_ARCH_HEENTRY + csrwr t0, LOONGARCH_CSR_EENTRY + +#if defined(CONFIG_CPU_HAS_FPU) + /* Save FPU context */ + csrrd t0, LOONGARCH_CSR_EUEN + ori t1, zero, CSR_EUEN_FPEN + and t2, t0, t1 + beqz t2, 1f + movfcsr2gr t3, fcsr0 + st.d t3, a2, VCPU_FCSR0 + + movcf2gr t3, $fcc0 + or t2, t3, zero + movcf2gr t3, $fcc1 + bstrins.d t2, t3, 0xf, 0x8 + movcf2gr t3, $fcc2 + bstrins.d t2, t3, 0x17, 0x10 + movcf2gr t3, $fcc3 + bstrins.d t2, t3, 0x1f, 0x18 + movcf2gr t3, $fcc4 + bstrins.d t2, t3, 0x27, 0x20 + movcf2gr t3, $fcc5 + bstrins.d t2, t3, 0x2f, 0x28 + movcf2gr t3, $fcc6 + bstrins.d t2, t3, 0x37, 0x30 + movcf2gr t3, $fcc7 + bstrins.d t2, t3, 0x3f, 0x38 + st.d t2, a2, VCPU_FCC + movgr2fcsr fcsr0, zero +1: +#endif + ld.d t0, a2, KVM_ARCH_HPGD + csrwr t0, LOONGARCH_CSR_PGDL + + /* Disable PVM bit for keeping from into guest */ + ori t0, zero, CSR_GSTAT_PVM + csrxchg zero, t0, LOONGARCH_CSR_GSTAT + /* Clear GTLBC.TGID field */ + csrrd t0, LOONGARCH_CSR_GTLBC + bstrins.w t0, zero, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT + csrwr t0, LOONGARCH_CSR_GTLBC + /* Enable Address Map mode */ + ori t0, zero, CONFIG_GUEST_CRMD + csrwr t0, LOONGARCH_CSR_CRMD + ld.d tp, a2, KVM_ARCH_HGP + ld.d sp, a2, KVM_ARCH_HSTACK + /* restore per cpu register */ + ld.d $r21, a2, KVM_ARCH_HPERCPU + addi.d sp, sp, -PT_SIZE + + /* Prepare handle exception */ + or a0, s0, zero + or a1, s1, zero + ld.d t8, a2, KVM_ARCH_HANDLE_EXIT + jirl ra,t8, 0 + + ori t0, zero, CSR_CRMD_IE + csrxchg zero, t0, LOONGARCH_CSR_CRMD + or a2, s1, zero + addi.d a2, a2, KVM_VCPU_ARCH + + andi t0, a0, RESUME_HOST + bnez t0, ret_to_host + + /* + * return to guest + * save per cpu register again, maybe switched to another cpu + */ + st.d $r21, a2, KVM_ARCH_HPERCPU + + /* Save kvm_vcpu to kscratch */ + csrwr s1, KVM_VCPU_KS + kvm_switch_to_guest a2 REG_A2 t0 t1 + +ret_to_host: + ld.d a2, a2, KVM_ARCH_HSTACK + addi.d a2, a2, -PT_SIZE + srai.w a3, a0, 2 + or a0, a3, zero + kvm_restore_host_gpr a2 + jirl zero, ra, 0 +SYM_CODE_END(kvm_vector_entry) +kvm_vector_entry_end: + +/* + * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu) + * + * @register_param: + * a0: kvm_run* run + * a1: kvm_vcpu* vcpu + */ +SYM_FUNC_START(kvm_enter_guest) + /* allocate space in stack bottom */ + addi.d a2, sp, -PT_SIZE + /* save host gprs */ + kvm_save_host_gpr a2 + + /* save host crmd,prmd csr to stack */ + csrrd a3, LOONGARCH_CSR_CRMD + st.d a3, a2, PT_CRMD + csrrd a3, LOONGARCH_CSR_PRMD + st.d a3, a2, PT_PRMD + + addi.d a2, a1, KVM_VCPU_ARCH + st.d sp, a2, KVM_ARCH_HSTACK + st.d tp, a2, KVM_ARCH_HGP + /* Save per cpu register */ + st.d $r21, a2, KVM_ARCH_HPERCPU + + /* Save kvm_vcpu to kscratch */ + csrwr a1, KVM_VCPU_KS + kvm_switch_to_guest a2 REG_A2 t0 t1 +SYM_FUNC_END(kvm_enter_guest) +kvm_enter_guest_end: + + .section ".rodata" +SYM_DATA(kvm_vector_size, + .quad kvm_vector_entry_end - kvm_vector_entry) +SYM_DATA(kvm_enter_guest_size, + .quad kvm_enter_guest_end - kvm_enter_guest) + + +SYM_FUNC_START(kvm_save_fpu) + fpu_save_double a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(kvm_save_fpu) + +SYM_FUNC_START(kvm_restore_fpu) + fpu_restore_double a0 t1 + jirl zero, ra, 0 +SYM_FUNC_END(kvm_restore_fpu) + +SYM_FUNC_START(kvm_restore_fcsr) + fpu_restore_csr a0 t1 + fpu_restore_cc a0 t1 t2 + + jirl zero, ra, 0 +SYM_FUNC_END(kvm_restore_fcsr) -- 2.31.1