This is a note to let you know that I've just added the patch titled
KVM: VMX: Flatten __vmx_vcpu_run()
to the 5.4-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary
The filename of the patch is:
kvm-vmx-flatten-__vmx_vcpu_run.patch
and it can be found in the queue-5.4 subdirectory.
If you, or anyone else, feels it should not be added to the stable tree,
please let <stable@xxxxxxxxxxxxxxx> know about it.
>From foo@baz Wed Oct 5 12:28:43 PM CEST 2022
From: Thadeu Lima de Souza Cascardo <cascardo@xxxxxxxxxxxxx>
Date: Mon, 3 Oct 2022 10:10:27 -0300
Subject: KVM: VMX: Flatten __vmx_vcpu_run()
To: stable@xxxxxxxxxxxxxxx
Cc: x86@xxxxxxxxxx, kvm@xxxxxxxxxxxxxxx, bp@xxxxxxxxx, pbonzini@xxxxxxxxxx, peterz@xxxxxxxxxxxxx, jpoimboe@xxxxxxxxxx
Message-ID: <20221003131038.12645-27-cascardo@xxxxxxxxxxxxx>
From: Josh Poimboeuf <jpoimboe@xxxxxxxxxx>
commit 8bd200d23ec42d66ccd517a72dd0b9cc6132d2fd upstream.
Move the vmx_vm{enter,exit}() functionality into __vmx_vcpu_run(). This
will make it easier to do the spec_ctrl handling before the first RET.
Signed-off-by: Josh Poimboeuf <jpoimboe@xxxxxxxxxx>
Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Signed-off-by: Borislav Petkov <bp@xxxxxxx>
[cascardo: remove ENDBR]
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@xxxxxxxxxxxxx>
Signed-off-by: Ben Hutchings <ben@xxxxxxxxxxxxxxx>
Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
[cascardo: no unwinding save/restore]
Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@xxxxxxxxxxxxx>
Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
---
arch/x86/kvm/vmx/vmenter.S | 114 ++++++++++++++-------------------------------
1 file changed, 37 insertions(+), 77 deletions(-)
--- a/arch/x86/kvm/vmx/vmenter.S
+++ b/arch/x86/kvm/vmx/vmenter.S
@@ -30,72 +30,6 @@
.text
/**
- * vmx_vmenter - VM-Enter the current loaded VMCS
- *
- * %RFLAGS.ZF: !VMCS.LAUNCHED, i.e. controls VMLAUNCH vs. VMRESUME
- *
- * Returns:
- * %RFLAGS.CF is set on VM-Fail Invalid
- * %RFLAGS.ZF is set on VM-Fail Valid
- * %RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
- *
- * Note that VMRESUME/VMLAUNCH fall-through and return directly if
- * they VM-Fail, whereas a successful VM-Enter + VM-Exit will jump
- * to vmx_vmexit.
- */
-ENTRY(vmx_vmenter)
- /* EFLAGS.ZF is set if VMCS.LAUNCHED == 0 */
- je 2f
-
-1: vmresume
- ret
-
-2: vmlaunch
- ret
-
-3: cmpb $0, kvm_rebooting
- je 4f
- ret
-4: ud2
-
- .pushsection .fixup, "ax"
-5: jmp 3b
- .popsection
-
- _ASM_EXTABLE(1b, 5b)
- _ASM_EXTABLE(2b, 5b)
-
-ENDPROC(vmx_vmenter)
-
-/**
- * vmx_vmexit - Handle a VMX VM-Exit
- *
- * Returns:
- * %RFLAGS.{CF,ZF} are cleared on VM-Success, i.e. VM-Exit
- *
- * This is vmx_vmenter's partner in crime. On a VM-Exit, control will jump
- * here after hardware loads the host's state, i.e. this is the destination
- * referred to by VMCS.HOST_RIP.
- */
-ENTRY(vmx_vmexit)
-#ifdef CONFIG_RETPOLINE
- ALTERNATIVE "jmp .Lvmexit_skip_rsb", "", X86_FEATURE_RETPOLINE
- /* Preserve guest's RAX, it's used to stuff the RSB. */
- push %_ASM_AX
-
- /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
- FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
-
- /* Clear RFLAGS.CF and RFLAGS.ZF to preserve VM-Exit, i.e. !VM-Fail. */
- or $1, %_ASM_AX
-
- pop %_ASM_AX
-.Lvmexit_skip_rsb:
-#endif
- ret
-ENDPROC(vmx_vmexit)
-
-/**
* __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
* @vmx: struct vcpu_vmx * (forwarded to vmx_update_host_rsp)
* @regs: unsigned long * (to guest registers)
@@ -127,8 +61,7 @@ ENTRY(__vmx_vcpu_run)
/* Copy @launched to BL, _ASM_ARG3 is volatile. */
mov %_ASM_ARG3B, %bl
- /* Adjust RSP to account for the CALL to vmx_vmenter(). */
- lea -WORD_SIZE(%_ASM_SP), %_ASM_ARG2
+ lea (%_ASM_SP), %_ASM_ARG2
call vmx_update_host_rsp
/* Load @regs to RAX. */
@@ -157,11 +90,25 @@ ENTRY(__vmx_vcpu_run)
/* Load guest RAX. This kills the @regs pointer! */
mov VCPU_RAX(%_ASM_AX), %_ASM_AX
- /* Enter guest mode */
- call vmx_vmenter
+ /* Check EFLAGS.ZF from 'testb' above */
+ je .Lvmlaunch
- /* Jump on VM-Fail. */
- jbe 2f
+/*
+ * If VMRESUME/VMLAUNCH and corresponding vmexit succeed, execution resumes at
+ * the 'vmx_vmexit' label below.
+ */
+.Lvmresume:
+ vmresume
+ jmp .Lvmfail
+
+.Lvmlaunch:
+ vmlaunch
+ jmp .Lvmfail
+
+ _ASM_EXTABLE(.Lvmresume, .Lfixup)
+ _ASM_EXTABLE(.Lvmlaunch, .Lfixup)
+
+SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL)
/* Temporarily save guest's RAX. */
push %_ASM_AX
@@ -188,9 +135,13 @@ ENTRY(__vmx_vcpu_run)
mov %r15, VCPU_R15(%_ASM_AX)
#endif
+ /* IMPORTANT: RSB must be stuffed before the first return. */
+ FILL_RETURN_BUFFER %_ASM_BX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE
+
/* Clear RAX to indicate VM-Exit (as opposed to VM-Fail). */
xor %eax, %eax
+.Lclear_regs:
/*
* Clear all general purpose registers except RSP and RAX to prevent
* speculative use of the guest's values, even those that are reloaded
@@ -200,7 +151,7 @@ ENTRY(__vmx_vcpu_run)
* free. RSP and RAX are exempt as RSP is restored by hardware during
* VM-Exit and RAX is explicitly loaded with 0 or 1 to return VM-Fail.
*/
-1: xor %ebx, %ebx
+ xor %ebx, %ebx
xor %ecx, %ecx
xor %edx, %edx
xor %esi, %esi
@@ -219,8 +170,8 @@ ENTRY(__vmx_vcpu_run)
/* "POP" @regs. */
add $WORD_SIZE, %_ASM_SP
- pop %_ASM_BX
+ pop %_ASM_BX
#ifdef CONFIG_X86_64
pop %r12
pop %r13
@@ -233,11 +184,20 @@ ENTRY(__vmx_vcpu_run)
pop %_ASM_BP
ret
- /* VM-Fail. Out-of-line to avoid a taken Jcc after VM-Exit. */
-2: mov $1, %eax
- jmp 1b
+.Lfixup:
+ cmpb $0, kvm_rebooting
+ jne .Lvmfail
+ ud2
+.Lvmfail:
+ /* VM-Fail: set return value to 1 */
+ mov $1, %eax
+ jmp .Lclear_regs
+
ENDPROC(__vmx_vcpu_run)
+
+.section .text, "ax"
+
/**
* vmread_error_trampoline - Trampoline from inline asm to vmread_error()
* @field: VMCS field encoding that failed
Patches currently in stable-queue which might be from cascardo@xxxxxxxxxxxxx are
queue-5.4/x86-speculation-disable-rrsba-behavior.patch
queue-5.4/kvm-vmx-flatten-__vmx_vcpu_run.patch
queue-5.4/x86-kvm-vmx-make-noinstr-clean.patch
queue-5.4/revert-x86-speculation-add-rsb-vm-exit-protections.patch
queue-5.4/kvm-vmx-fix-ibrs-handling-after-vmexit.patch
queue-5.4/kvm-vmx-prevent-guest-rsb-poisoning-attacks-with-eibrs.patch
queue-5.4/kvm-nvmx-use-__vmx_vcpu_run-in-nested_vmx_check_vmentry_hw.patch
queue-5.4/x86-bugs-keep-a-per-cpu-ia32_spec_ctrl-value.patch
queue-5.4/x86-cpu-amd-enumerate-btc_no.patch
queue-5.4/x86-speculation-fix-firmware-entry-spec_ctrl-handling.patch
queue-5.4/x86-speculation-add-spectre_v2-ibrs-option-to-support-kernel-ibrs.patch
queue-5.4/x86-cpu-add-consistent-cpu-match-macros.patch
queue-5.4/x86-speculation-remove-x86_spec_ctrl_mask.patch
queue-5.4/x86-bugs-add-cannon-lake-to-retbleed-affected-cpu-list.patch
queue-5.4/x86-bugs-warn-when-ibrs-mitigation-is-selected-on-enhanced-ibrs-parts.patch
queue-5.4/x86-speculation-fill-rsb-on-vmexit-for-ibrs.patch
queue-5.4/x86-cpu-add-a-steppings-field-to-struct-x86_cpu_id.patch
queue-5.4/kvm-vmx-convert-launched-argument-to-flags.patch
queue-5.4/x86-common-stamp-out-the-stepping-madness.patch
queue-5.4/x86-bugs-split-spectre_v2_select_mitigation-and-spectre_v2_user_select_mitigation.patch
queue-5.4/x86-bugs-report-intel-retbleed-vulnerability.patch
queue-5.4/x86-speculation-change-fill_return_buffer-to-work-with-objtool.patch
queue-5.4/x86-cpufeatures-move-retpoline-flags-to-word-11.patch
queue-5.4/x86-speculation-fix-spec_ctrl-write-on-smt-state-change.patch
queue-5.4/kvm-vmx-use-test-reg-reg-instead-of-cmp-0-reg-in-vmenter.s.patch
queue-5.4/x86-bugs-optimize-spec_ctrl-msr-writes.patch
queue-5.4/x86-bugs-report-amd-retbleed-vulnerability.patch
queue-5.4/x86-speculation-fix-rsb-filling-with-config_retpoline-n.patch
queue-5.4/intel_idle-disable-ibrs-during-long-idle.patch
queue-5.4/x86-speculation-use-declare_per_cpu-for-x86_spec_ctrl_current.patch
queue-5.4/x86-entry-remove-skip_r11rcx.patch
queue-5.4/x86-speculation-use-cached-host-spec_ctrl-value-for-guest-entry-exit.patch
queue-5.4/x86-devicetable-move-x86-specific-macro-out-of-generic-code.patch
queue-5.4/x86-bugs-add-amd-retbleed-boot-parameter.patch
queue-5.4/x86-entry-add-kernel-ibrs-implementation.patch
queue-5.4/revert-x86-cpu-add-a-steppings-field-to-struct-x86_cpu_id.patch
queue-5.4/x86-speculation-add-rsb-vm-exit-protections.patch
