This is a note to let you know that I've just added the patch titled
x86/bugs: Add asm helpers for executing VERW
to the 6.1-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary
The filename of the patch is:
x86-bugs-add-asm-helpers-for-executing-verw.patch
and it can be found in the queue-6.1 subdirectory.
If you, or anyone else, feels it should not be added to the stable tree,
please let <stable@xxxxxxxxxxxxxxx> know about it.
>From stable+bounces-25864-greg=kroah.com@xxxxxxxxxxxxxxx Mon Mar 4 10:25:18 2024
From: Pawan Gupta <pawan.kumar.gupta@xxxxxxxxxxxxxxx>
Date: Mon, 4 Mar 2024 01:23:53 -0800
Subject: x86/bugs: Add asm helpers for executing VERW
To: stable@xxxxxxxxxxxxxxx
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>, Alyssa Milburn <alyssa.milburn@xxxxxxxxx>, Andrew Cooper <andrew.cooper3@xxxxxxxxxx>, Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Message-ID: <20240304-delay-verw-backport-6-1-y-v2-1-bf4bce517d60@xxxxxxxxxxxxxxx>
Content-Disposition: inline
From: Pawan Gupta <pawan.kumar.gupta@xxxxxxxxxxxxxxx>
commit baf8361e54550a48a7087b603313ad013cc13386 upstream.
MDS mitigation requires clearing the CPU buffers before returning to
user. This needs to be done late in the exit-to-user path. Current
location of VERW leaves a possibility of kernel data ending up in CPU
buffers for memory accesses done after VERW such as:
1. Kernel data accessed by an NMI between VERW and return-to-user can
remain in CPU buffers since NMI returning to kernel does not
execute VERW to clear CPU buffers.
2. Alyssa reported that after VERW is executed,
CONFIG_GCC_PLUGIN_STACKLEAK=y scrubs the stack used by a system
call. Memory accesses during stack scrubbing can move kernel stack
contents into CPU buffers.
3. When caller saved registers are restored after a return from
function executing VERW, the kernel stack accesses can remain in
CPU buffers(since they occur after VERW).
To fix this VERW needs to be moved very late in exit-to-user path.
In preparation for moving VERW to entry/exit asm code, create macros
that can be used in asm. Also make VERW patching depend on a new feature
flag X86_FEATURE_CLEAR_CPU_BUF.
[pawan: - Runtime patch jmp instead of verw in macro CLEAR_CPU_BUFFERS
due to lack of relative addressing support for relocations
in kernels < v6.5.
- Add UNWIND_HINT_EMPTY to avoid warning:
arch/x86/entry/entry.o: warning: objtool: mds_verw_sel+0x0: unreachable instruction]
Reported-by: Alyssa Milburn <alyssa.milburn@xxxxxxxxx>
Suggested-by: Andrew Cooper <andrew.cooper3@xxxxxxxxxx>
Suggested-by: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@xxxxxxxxxxxxxxx>
Signed-off-by: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Link: https://lore.kernel.org/all/20240213-delay-verw-v8-1-a6216d83edb7%40linux.intel.com
Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
---
arch/x86/entry/entry.S | 23 +++++++++++++++++++++++
arch/x86/include/asm/cpufeatures.h | 2 +-
arch/x86/include/asm/nospec-branch.h | 15 +++++++++++++++
3 files changed, 39 insertions(+), 1 deletion(-)
--- a/arch/x86/entry/entry.S
+++ b/arch/x86/entry/entry.S
@@ -6,6 +6,9 @@
#include <linux/linkage.h>
#include <asm/export.h>
#include <asm/msr-index.h>
+#include <asm/unwind_hints.h>
+#include <asm/segment.h>
+#include <asm/cache.h>
.pushsection .noinstr.text, "ax"
@@ -20,3 +23,23 @@ SYM_FUNC_END(entry_ibpb)
EXPORT_SYMBOL_GPL(entry_ibpb);
.popsection
+
+/*
+ * Define the VERW operand that is disguised as entry code so that
+ * it can be referenced with KPTI enabled. This ensure VERW can be
+ * used late in exit-to-user path after page tables are switched.
+ */
+.pushsection .entry.text, "ax"
+
+.align L1_CACHE_BYTES, 0xcc
+SYM_CODE_START_NOALIGN(mds_verw_sel)
+ UNWIND_HINT_EMPTY
+ ANNOTATE_NOENDBR
+ .word __KERNEL_DS
+.align L1_CACHE_BYTES, 0xcc
+SYM_CODE_END(mds_verw_sel);
+/* For KVM */
+EXPORT_SYMBOL_GPL(mds_verw_sel);
+
+.popsection
+
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -304,7 +304,7 @@
#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */
#define X86_FEATURE_USE_IBPB_FW (11*32+16) /* "" Use IBPB during runtime firmware calls */
#define X86_FEATURE_RSB_VMEXIT_LITE (11*32+17) /* "" Fill RSB on VM exit when EIBRS is enabled */
-
+#define X86_FEATURE_CLEAR_CPU_BUF (11*32+18) /* "" Clear CPU buffers using VERW */
#define X86_FEATURE_MSR_TSX_CTRL (11*32+20) /* "" MSR IA32_TSX_CTRL (Intel) implemented */
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -194,6 +194,19 @@
#endif
.endm
+/*
+ * Macro to execute VERW instruction that mitigate transient data sampling
+ * attacks such as MDS. On affected systems a microcode update overloaded VERW
+ * instruction to also clear the CPU buffers. VERW clobbers CFLAGS.ZF.
+ *
+ * Note: Only the memory operand variant of VERW clears the CPU buffers.
+ */
+.macro CLEAR_CPU_BUFFERS
+ ALTERNATIVE "jmp .Lskip_verw_\@", "", X86_FEATURE_CLEAR_CPU_BUF
+ verw _ASM_RIP(mds_verw_sel)
+.Lskip_verw_\@:
+.endm
+
#else /* __ASSEMBLY__ */
#define ANNOTATE_RETPOLINE_SAFE \
@@ -375,6 +388,8 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_cond_
DECLARE_STATIC_KEY_FALSE(mmio_stale_data_clear);
+extern u16 mds_verw_sel;
+
#include <asm/segment.h>
/**
Patches currently in stable-queue which might be from kroah.com@xxxxxxxxxxxxxxx are
queue-6.1/x86-bugs-use-alternative-instead-of-mds_user_clear-static-key.patch
queue-6.1/x86-decompressor-avoid-magic-offsets-for-efi-handover-entrypoint.patch
queue-6.1/x86-efistub-avoid-placing-the-kernel-below-load_physical_addr.patch
queue-6.1/x86-efistub-avoid-legacy-decompressor-when-doing-efi-boot.patch
queue-6.1/kvm-vmx-move-verw-closer-to-vmentry-for-mds-mitigation.patch
queue-6.1/x86-efistub-perform-4-5-level-paging-switch-from-the-stub.patch
queue-6.1/x86-entry_32-add-verw-just-before-userspace-transition.patch
queue-6.1/x86-boot-rename-conflicting-boot_params-pointer-to-boot_params_ptr.patch
queue-6.1/x86-efistub-perform-snp-feature-test-while-running-in-the-firmware.patch
queue-6.1/x86-efistub-simplify-and-clean-up-handover-entry-code.patch
queue-6.1/efi-libstub-add-memory-attribute-protocol-definitions.patch
queue-6.1/x86-entry_64-add-verw-just-before-userspace-transition.patch
queue-6.1/efi-libstub-add-limit-argument-to-efi_random_alloc.patch
queue-6.1/efi-x86-fix-the-missing-kaslr_flag-bit-in-boot_params-hdr.loadflags.patch
queue-6.1/kvm-vmx-use-bt-jnc-i.e.-eflags.cf-to-select-vmresume-vs.-vmlaunch.patch
queue-6.1/arm64-efi-limit-allocations-to-48-bit-addressable-physical-region.patch
queue-6.1/x86-efistub-prefer-efi-memory-attributes-protocol-over-dxe-services.patch
queue-6.1/efi-efivars-prevent-double-registration.patch
queue-6.1/x86-boot-efistub-assign-global-boot_params-variable.patch
queue-6.1/x86-efistub-clear-bss-in-efi-handover-protocol-entrypoint.patch
queue-6.1/x86-decompressor-factor-out-kernel-decompression-and-relocation.patch
queue-6.1/efi-x86-avoid-physical-kaslr-on-older-dell-systems.patch
queue-6.1/x86-bugs-add-asm-helpers-for-executing-verw.patch
