This is a note to let you know that I've just added the patch titled x86/bugs: Fix the SRSO mitigation on Zen3/4 to the 6.6-stable tree which can be found at: http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary The filename of the patch is: x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch and it can be found in the queue-6.6 subdirectory. If you, or anyone else, feels it should not be added to the stable tree, please let <stable@xxxxxxxxxxxxxxx> know about it. >From 5ec47ec6bd0952874bd60b3f8e7833eb2b9cdcbb Mon Sep 17 00:00:00 2001 From: "Borislav Petkov (AMD)" <bp@xxxxxxxxx> Date: Thu, 28 Mar 2024 13:59:05 +0100 Subject: x86/bugs: Fix the SRSO mitigation on Zen3/4 From: "Borislav Petkov (AMD)" <bp@xxxxxxxxx> Commit 4535e1a4174c4111d92c5a9a21e542d232e0fcaa upstream. The original version of the mitigation would patch in the calls to the untraining routines directly. That is, the alternative() in UNTRAIN_RET will patch in the CALL to srso_alias_untrain_ret() directly. However, even if commit e7c25c441e9e ("x86/cpu: Cleanup the untrain mess") meant well in trying to clean up the situation, due to micro- architectural reasons, the untraining routine srso_alias_untrain_ret() must be the target of a CALL instruction and not of a JMP instruction as it is done now. Reshuffle the alternative macros to accomplish that. Fixes: e7c25c441e9e ("x86/cpu: Cleanup the untrain mess") Signed-off-by: Borislav Petkov (AMD) <bp@xxxxxxxxx> Reviewed-by: Ingo Molnar <mingo@xxxxxxxxxx> Cc: stable@xxxxxxxxxx Signed-off-by: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx> Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> --- arch/x86/include/asm/asm-prototypes.h | 1 + arch/x86/include/asm/nospec-branch.h | 21 ++++++++++++++++----- arch/x86/lib/retpoline.S | 10 +++++----- 3 files changed, 22 insertions(+), 10 deletions(-) --- a/arch/x86/include/asm/asm-prototypes.h +++ b/arch/x86/include/asm/asm-prototypes.h @@ -13,6 +13,7 @@ #include <asm/preempt.h> #include <asm/asm.h> #include <asm/gsseg.h> +#include <asm/nospec-branch.h> #ifndef CONFIG_X86_CMPXCHG64 extern void cmpxchg8b_emu(void); --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -271,11 +271,20 @@ .Lskip_rsb_\@: .endm +/* + * The CALL to srso_alias_untrain_ret() must be patched in directly at + * the spot where untraining must be done, ie., srso_alias_untrain_ret() + * must be the target of a CALL instruction instead of indirectly + * jumping to a wrapper which then calls it. Therefore, this macro is + * called outside of __UNTRAIN_RET below, for the time being, before the + * kernel can support nested alternatives with arbitrary nesting. + */ +.macro CALL_UNTRAIN_RET #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO) -#define CALL_UNTRAIN_RET "call entry_untrain_ret" -#else -#define CALL_UNTRAIN_RET "" + ALTERNATIVE_2 "", "call entry_untrain_ret", X86_FEATURE_UNRET, \ + "call srso_alias_untrain_ret", X86_FEATURE_SRSO_ALIAS #endif +.endm /* * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the @@ -291,8 +300,8 @@ .macro __UNTRAIN_RET ibpb_feature, call_depth_insns #if defined(CONFIG_RETHUNK) || defined(CONFIG_CPU_IBPB_ENTRY) VALIDATE_UNRET_END - ALTERNATIVE_3 "", \ - CALL_UNTRAIN_RET, X86_FEATURE_UNRET, \ + CALL_UNTRAIN_RET + ALTERNATIVE_2 "", \ "call entry_ibpb", \ibpb_feature, \ __stringify(\call_depth_insns), X86_FEATURE_CALL_DEPTH #endif @@ -351,6 +360,8 @@ extern void retbleed_return_thunk(void); static inline void retbleed_return_thunk(void) {} #endif +extern void srso_alias_untrain_ret(void); + #ifdef CONFIG_CPU_SRSO extern void srso_return_thunk(void); extern void srso_alias_return_thunk(void); --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -218,10 +218,12 @@ SYM_CODE_START(srso_return_thunk) SYM_CODE_END(srso_return_thunk) #define JMP_SRSO_UNTRAIN_RET "jmp srso_untrain_ret" -#define JMP_SRSO_ALIAS_UNTRAIN_RET "jmp srso_alias_untrain_ret" #else /* !CONFIG_CPU_SRSO */ #define JMP_SRSO_UNTRAIN_RET "ud2" -#define JMP_SRSO_ALIAS_UNTRAIN_RET "ud2" +/* Dummy for the alternative in CALL_UNTRAIN_RET. */ +SYM_CODE_START(srso_alias_untrain_ret) + RET +SYM_FUNC_END(srso_alias_untrain_ret) #endif /* CONFIG_CPU_SRSO */ #ifdef CONFIG_CPU_UNRET_ENTRY @@ -314,9 +316,7 @@ __EXPORT_THUNK(retbleed_untrain_ret) #if defined(CONFIG_CPU_UNRET_ENTRY) || defined(CONFIG_CPU_SRSO) SYM_FUNC_START(entry_untrain_ret) - ALTERNATIVE_2 JMP_RETBLEED_UNTRAIN_RET, \ - JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO, \ - JMP_SRSO_ALIAS_UNTRAIN_RET, X86_FEATURE_SRSO_ALIAS + ALTERNATIVE JMP_RETBLEED_UNTRAIN_RET, JMP_SRSO_UNTRAIN_RET, X86_FEATURE_SRSO SYM_FUNC_END(entry_untrain_ret) __EXPORT_THUNK(entry_untrain_ret) Patches currently in stable-queue which might be from bp@xxxxxxxxx are queue-6.6/x86-cpu-amd-get-rid-of-amd_erratum_1054.patch queue-6.6/drm-i915-pre-populate-the-cursor-physical-dma-addres.patch queue-6.6/x86-srso-improve-i-cache-locality-for-alias-mitigation.patch queue-6.6/x86-bugs-fix-the-srso-mitigation-on-zen3-4.patch queue-6.6/x86-srso-disentangle-rethunk-dependent-options.patch queue-6.6/x86-cpu-amd-move-the-div0-bug-detection-to-the-zen1-.patch queue-6.6/x86-cpu-amd-carve-out-the-erratum-1386-fix.patch queue-6.6/x86-nospec-refactor-untrain_ret.patch queue-6.6/mm-treewide-replace-pud_large-with-pud_leaf.patch queue-6.6/x86-cpu-amd-add-x86_feature_zen1.patch queue-6.6/x86-cpu-amd-add-zenx-generations-flags.patch queue-6.6/x86-cpu-amd-move-erratum-1076-fix-into-the-zen1-init.patch queue-6.6/x86-cpu-amd-move-zenbleed-check-to-the-zen2-init-fun.patch