This is a note to let you know that I've just added the patch titled x86: Add magic AMD return-thunk to the 5.10-stable tree which can be found at: http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary The filename of the patch is: x86-add-magic-amd-return-thunk.patch and it can be found in the queue-5.10 subdirectory. If you, or anyone else, feels it should not be added to the stable tree, please let <stable@xxxxxxxxxxxxxxx> know about it. >From foo@baz Tue Jul 12 05:07:35 PM CEST 2022 From: Peter Zijlstra <peterz@xxxxxxxxxxxxx> Date: Tue, 14 Jun 2022 23:15:48 +0200 Subject: x86: Add magic AMD return-thunk From: Peter Zijlstra <peterz@xxxxxxxxxxxxx> commit a149180fbcf336e97ce4eb2cdc13672727feb94d upstream. Note: needs to be in a section distinct from Retpolines such that the Retpoline RET substitution cannot possibly use immediate jumps. ORC unwinding for zen_untrain_ret() and __x86_return_thunk() is a little tricky but works due to the fact that zen_untrain_ret() doesn't have any stack ops and as such will emit a single ORC entry at the start (+0x3f). Meanwhile, unwinding an IP, including the __x86_return_thunk() one (+0x40) will search for the largest ORC entry smaller or equal to the IP, these will find the one ORC entry (+0x3f) and all works. [ Alexandre: SVM part. ] [ bp: Build fix, massages. ] Suggested-by: Andrew Cooper <Andrew.Cooper3@xxxxxxxxxx> Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx> Signed-off-by: Borislav Petkov <bp@xxxxxxx> Reviewed-by: Josh Poimboeuf <jpoimboe@xxxxxxxxxx> Signed-off-by: Borislav Petkov <bp@xxxxxxx> [cascardo: conflicts at arch/x86/entry/entry_64_compat.S] [cascardo: there is no ANNOTATE_NOENDBR] [cascardo: objtool commit 34c861e806478ac2ea4032721defbf1d6967df08 missing] [cascardo: conflict fixup] Signed-off-by: Thadeu Lima de Souza Cascardo <cascardo@xxxxxxxxxxxxx> [bwh: Backported to 5.10: SEV-ES is not supported, so drop the change in arch/x86/kvm/svm/vmenter.S] Signed-off-by: Ben Hutchings <ben@xxxxxxxxxxxxxxx> Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> --- arch/x86/entry/entry_64.S | 6 ++ arch/x86/entry/entry_64_compat.S | 4 + arch/x86/include/asm/cpufeatures.h | 1 arch/x86/include/asm/disabled-features.h | 3 - arch/x86/include/asm/nospec-branch.h | 17 ++++++++ arch/x86/kernel/vmlinux.lds.S | 2 arch/x86/kvm/svm/vmenter.S | 9 ++++ arch/x86/lib/retpoline.S | 63 +++++++++++++++++++++++++++++-- tools/objtool/check.c | 20 ++++++++- 9 files changed, 117 insertions(+), 8 deletions(-) --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -102,6 +102,7 @@ SYM_CODE_START(entry_SYSCALL_64) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL) + UNTRAIN_RET /* Construct struct pt_regs on stack */ pushq $__USER_DS /* pt_regs->ss */ @@ -675,6 +676,7 @@ native_irq_return_ldt: pushq %rdi /* Stash user RDI */ swapgs /* to kernel GS */ SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */ + UNTRAIN_RET movq PER_CPU_VAR(espfix_waddr), %rdi movq %rax, (0*8)(%rdi) /* user RAX */ @@ -910,6 +912,7 @@ SYM_CODE_START_LOCAL(paranoid_entry) * be retrieved from a kernel internal table. */ SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14 + UNTRAIN_RET /* * Handling GSBASE depends on the availability of FSGSBASE. @@ -1022,6 +1025,7 @@ SYM_CODE_START_LOCAL(error_entry) FENCE_SWAPGS_USER_ENTRY /* We have user CR3. Change to kernel CR3. */ SWITCH_TO_KERNEL_CR3 scratch_reg=%rax + UNTRAIN_RET .Lerror_entry_from_usermode_after_swapgs: /* Put us onto the real thread stack. */ @@ -1077,6 +1081,7 @@ SYM_CODE_START_LOCAL(error_entry) SWAPGS FENCE_SWAPGS_USER_ENTRY SWITCH_TO_KERNEL_CR3 scratch_reg=%rax + UNTRAIN_RET /* * Pretend that the exception came from user mode: set up pt_regs @@ -1171,6 +1176,7 @@ SYM_CODE_START(asm_exc_nmi) movq %rsp, %rdx movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp UNWIND_HINT_IRET_REGS base=%rdx offset=8 + UNTRAIN_RET pushq 5*8(%rdx) /* pt_regs->ss */ pushq 4*8(%rdx) /* pt_regs->rsp */ pushq 3*8(%rdx) /* pt_regs->flags */ --- a/arch/x86/entry/entry_64_compat.S +++ b/arch/x86/entry/entry_64_compat.S @@ -14,6 +14,7 @@ #include <asm/irqflags.h> #include <asm/asm.h> #include <asm/smap.h> +#include <asm/nospec-branch.h> #include <linux/linkage.h> #include <linux/err.h> @@ -71,6 +72,7 @@ SYM_CODE_START(entry_SYSENTER_compat) pushq $__USER32_CS /* pt_regs->cs */ pushq $0 /* pt_regs->ip = 0 (placeholder) */ SYM_INNER_LABEL(entry_SYSENTER_compat_after_hwframe, SYM_L_GLOBAL) + UNTRAIN_RET /* * User tracing code (ptrace or signal handlers) might assume that @@ -211,6 +213,7 @@ SYM_CODE_START(entry_SYSCALL_compat) movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp SYM_INNER_LABEL(entry_SYSCALL_compat_safe_stack, SYM_L_GLOBAL) + UNTRAIN_RET /* Construct struct pt_regs on stack */ pushq $__USER32_DS /* pt_regs->ss */ @@ -377,6 +380,7 @@ SYM_CODE_START(entry_INT80_compat) pushq (%rdi) /* pt_regs->di */ .Lint80_keep_stack: + UNTRAIN_RET pushq %rsi /* pt_regs->si */ xorl %esi, %esi /* nospec si */ pushq %rdx /* pt_regs->dx */ --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -297,6 +297,7 @@ #define X86_FEATURE_RETPOLINE (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ #define X86_FEATURE_RETPOLINE_LFENCE (11*32+13) /* "" Use LFENCE for Spectre variant 2 */ #define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */ +#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX512_BF16 (12*32+ 5) /* AVX512 BFLOAT16 instructions */ --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -61,7 +61,8 @@ #else # define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) | \ (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31)) | \ - (1 << (X86_FEATURE_RETHUNK & 31))) + (1 << (X86_FEATURE_RETHUNK & 31)) | \ + (1 << (X86_FEATURE_UNRET & 31))) #endif /* Force disable because it's broken beyond repair */ --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -112,6 +112,22 @@ #endif .endm +/* + * Mitigate RETBleed for AMD/Hygon Zen uarch. Requires KERNEL CR3 because the + * return thunk isn't mapped into the userspace tables (then again, AMD + * typically has NO_MELTDOWN). + * + * Doesn't clobber any registers but does require a stable stack. + * + * As such, this must be placed after every *SWITCH_TO_KERNEL_CR3 at a point + * where we have a stack but before any RET instruction. + */ +.macro UNTRAIN_RET +#ifdef CONFIG_RETPOLINE + ALTERNATIVE "", "call zen_untrain_ret", X86_FEATURE_UNRET +#endif +.endm + #else /* __ASSEMBLY__ */ #define ANNOTATE_RETPOLINE_SAFE \ @@ -121,6 +137,7 @@ ".popsection\n\t" extern void __x86_return_thunk(void); +extern void zen_untrain_ret(void); #ifdef CONFIG_RETPOLINE --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -142,7 +142,7 @@ SECTIONS #ifdef CONFIG_RETPOLINE __indirect_thunk_start = .; - *(.text.__x86.indirect_thunk) + *(.text.__x86.*) __indirect_thunk_end = .; #endif } :text =0xcccc --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -129,6 +129,15 @@ SYM_FUNC_START(__svm_vcpu_run) #endif /* + * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be + * untrained as soon as we exit the VM and are back to the + * kernel. This should be done before re-enabling interrupts + * because interrupt handlers won't sanitize 'ret' if the return is + * from the kernel. + */ + UNTRAIN_RET + + /* * Clear all general purpose registers except RSP and RAX to prevent * speculative use of the guest's values, even those that are reloaded * via the stack. In theory, an L1 cache miss when restoring registers --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -71,10 +71,67 @@ SYM_CODE_END(__x86_indirect_thunk_array) * This function name is magical and is used by -mfunction-return=thunk-extern * for the compiler to generate JMPs to it. */ -SYM_CODE_START(__x86_return_thunk) - UNWIND_HINT_EMPTY + .section .text.__x86.return_thunk + +/* + * Safety details here pertain to the AMD Zen{1,2} microarchitecture: + * 1) The RET at __x86_return_thunk must be on a 64 byte boundary, for + * alignment within the BTB. + * 2) The instruction at zen_untrain_ret must contain, and not + * end with, the 0xc3 byte of the RET. + * 3) STIBP must be enabled, or SMT disabled, to prevent the sibling thread + * from re-poisioning the BTB prediction. + */ + .align 64 + .skip 63, 0xcc +SYM_FUNC_START_NOALIGN(zen_untrain_ret); + + /* + * As executed from zen_untrain_ret, this is: + * + * TEST $0xcc, %bl + * LFENCE + * JMP __x86_return_thunk + * + * Executing the TEST instruction has a side effect of evicting any BTB + * prediction (potentially attacker controlled) attached to the RET, as + * __x86_return_thunk + 1 isn't an instruction boundary at the moment. + */ + .byte 0xf6 + + /* + * As executed from __x86_return_thunk, this is a plain RET. + * + * As part of the TEST above, RET is the ModRM byte, and INT3 the imm8. + * + * We subsequently jump backwards and architecturally execute the RET. + * This creates a correct BTB prediction (type=ret), but in the + * meantime we suffer Straight Line Speculation (because the type was + * no branch) which is halted by the INT3. + * + * With SMT enabled and STIBP active, a sibling thread cannot poison + * RET's prediction to a type of its choice, but can evict the + * prediction due to competitive sharing. If the prediction is + * evicted, __x86_return_thunk will suffer Straight Line Speculation + * which will be contained safely by the INT3. + */ +SYM_INNER_LABEL(__x86_return_thunk, SYM_L_GLOBAL) ret int3 SYM_CODE_END(__x86_return_thunk) -__EXPORT_THUNK(__x86_return_thunk) + /* + * Ensure the TEST decoding / BTB invalidation is complete. + */ + lfence + + /* + * Jump back and execute the RET in the middle of the TEST instruction. + * INT3 is for SLS protection. + */ + jmp __x86_return_thunk + int3 +SYM_FUNC_END(zen_untrain_ret) +__EXPORT_THUNK(zen_untrain_ret) + +EXPORT_SYMBOL(__x86_return_thunk) --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1082,7 +1082,7 @@ static void add_retpoline_call(struct ob annotate_call_site(file, insn, false); } -static void add_return_call(struct objtool_file *file, struct instruction *insn) +static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) { /* * Return thunk tail calls are really just returns in disguise, @@ -1092,7 +1092,7 @@ static void add_return_call(struct objto insn->retpoline_safe = true; /* Skip the non-text sections, specially .discard ones */ - if (insn->sec->text) + if (add && insn->sec->text) list_add_tail(&insn->call_node, &file->return_thunk_list); } @@ -1121,7 +1121,7 @@ static int add_jump_destinations(struct add_retpoline_call(file, insn); continue; } else if (reloc->sym->return_thunk) { - add_return_call(file, insn); + add_return_call(file, insn, true); continue; } else if (insn->func) { /* internal or external sibling call (with reloc) */ @@ -1138,6 +1138,7 @@ static int add_jump_destinations(struct insn->jump_dest = find_insn(file, dest_sec, dest_off); if (!insn->jump_dest) { + struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); /* * This is a special case where an alt instruction @@ -1147,6 +1148,19 @@ static int add_jump_destinations(struct if (!strcmp(insn->sec->name, ".altinstr_replacement")) continue; + /* + * This is a special case for zen_untrain_ret(). + * It jumps to __x86_return_thunk(), but objtool + * can't find the thunk's starting RET + * instruction, because the RET is also in the + * middle of another instruction. Objtool only + * knows about the outer instruction. + */ + if (sym && sym->return_thunk) { + add_return_call(file, insn, false); + continue; + } + WARN_FUNC("can't find jump dest instruction at %s+0x%lx", insn->sec, insn->offset, dest_sec->name, dest_off); Patches currently in stable-queue which might be from peterz@xxxxxxxxxxxxx are queue-5.10/objtool-cache-instruction-relocs.patch queue-5.10/x86-sev-avoid-using-__x86_return_thunk.patch queue-5.10/objtool-add-elf_create_undef_symbol.patch queue-5.10/x86-ftrace-use-alternative-ret-encoding.patch queue-5.10/objtool-re-add-unwind_hint_-save_restore.patch queue-5.10/x86-bugs-add-retbleed-ibpb.patch queue-5.10/x86-bugs-enable-stibp-for-jmp2ret.patch queue-5.10/x86-retpoline-cleanup-some-ifdefery.patch queue-5.10/objtool-handle-__sanitize_cov-tail-calls.patch queue-5.10/x86-prepare-asm-files-for-straight-line-speculation.patch queue-5.10/kvm-vmx-flatten-__vmx_vcpu_run.patch queue-5.10/x86-kvm-vmx-make-noinstr-clean.patch queue-5.10/objtool-x86-replace-alternatives-with-.retpoline_sites.patch queue-5.10/objtool-skip-magical-retpoline-.altinstr_replacement.patch queue-5.10/x86-retbleed-add-fine-grained-kconfig-knobs.patch queue-5.10/x86-cpu-amd-add-spectral-chicken.patch queue-5.10/objtool-add-straight-line-speculation-validation.patch queue-5.10/kvm-vmx-fix-ibrs-handling-after-vmexit.patch queue-5.10/kvm-vmx-prevent-guest-rsb-poisoning-attacks-with-eibrs.patch queue-5.10/x86-vsyscall_emu-64-don-t-use-ret-in-vsyscall-emulation.patch queue-5.10/tools-arch-update-arch-x86-lib-mem-cpy-set-_64.s-copies-used-in-perf-bench-mem-memcpy.patch queue-5.10/x86-add-straight-line-speculation-mitigation.patch queue-5.10/x86-add-magic-amd-return-thunk.patch queue-5.10/x86-bugs-keep-a-per-cpu-ia32_spec_ctrl-value.patch queue-5.10/x86-alternatives-optimize-optimize_nops.patch queue-5.10/x86-objtool-create-.return_sites.patch queue-5.10/crypto-x86-poly1305-fixup-sls.patch queue-5.10/x86-alternative-handle-jcc-__x86_indirect_thunk_-reg.patch queue-5.10/x86-kvm-fix-setcc-emulation-for-return-thunks.patch queue-5.10/objtool-fix-objtool-regression-on-x32-systems.patch queue-5.10/x86-alternative-relax-text_poke_bp-constraint.patch queue-5.10/x86-retpoline-swizzle-retpoline-thunk.patch queue-5.10/objtool-rework-the-elf_rebuild_reloc_section-logic.patch queue-5.10/x86-speculation-fix-firmware-entry-spec_ctrl-handling.patch queue-5.10/x86-retpoline-remove-unused-replacement-symbols.patch queue-5.10/objtool-fix-symbol-creation.patch queue-5.10/x86-speculation-add-spectre_v2-ibrs-option-to-support-kernel-ibrs.patch queue-5.10/bpf-x86-respect-x86_feature_retpoline.patch queue-5.10/objtool-fix-type-of-reloc-addend.patch queue-5.10/objtool-x86-rewrite-retpoline-thunk-calls.patch queue-5.10/x86-undo-return-thunk-damage.patch queue-5.10/x86-prepare-inline-asm-for-straight-line-speculation.patch queue-5.10/x86-alternative-support-alternative_ternary.patch queue-5.10/kvm-emulate-fix-setcc-emulation-function-offsets-with-sls.patch queue-5.10/objtool-handle-per-arch-retpoline-naming.patch queue-5.10/x86-retpoline-create-a-retpoline-thunk-array.patch queue-5.10/x86-retpoline-simplify-retpolines.patch queue-5.10/x86-asm-fix-register-order.patch queue-5.10/x86-speculation-fill-rsb-on-vmexit-for-ibrs.patch queue-5.10/objtool-add-entry-unret-validation.patch queue-5.10/objtool-keep-track-of-retpoline-call-sites.patch queue-5.10/kvm-vmx-convert-launched-argument-to-flags.patch queue-5.10/objtool-add-elf_create_reloc-helper.patch queue-5.10/objtool-make-.altinstructions-section-entry-size-consistent.patch queue-5.10/x86-bpf-use-alternative-ret-encoding.patch queue-5.10/x86-common-stamp-out-the-stepping-madness.patch queue-5.10/x86-bugs-split-spectre_v2_select_mitigation-and-spectre_v2_user_select_mitigation.patch queue-5.10/x86-bugs-report-intel-retbleed-vulnerability.patch queue-5.10/bpf-x86-simplify-computing-label-offsets.patch queue-5.10/x86-cpufeatures-move-retpoline-flags-to-word-11.patch queue-5.10/x86-speculation-fix-spec_ctrl-write-on-smt-state-change.patch queue-5.10/x86-retpoline-use-mfunction-return.patch queue-5.10/x86-xen-rename-sys-entry-points.patch queue-5.10/objtool-only-rewrite-unconditional-retpoline-thunk-calls.patch queue-5.10/x86-bugs-optimize-spec_ctrl-msr-writes.patch queue-5.10/x86-alternative-optimize-single-byte-nops-at-an-arbitrary-position.patch queue-5.10/objtool-fix-code-relocs-vs-weak-symbols.patch queue-5.10/x86-bugs-report-amd-retbleed-vulnerability.patch queue-5.10/x86-static_call-use-alternative-ret-encoding.patch queue-5.10/x86-speculation-fix-rsb-filling-with-config_retpoline-n.patch queue-5.10/x86-asm-fixup-odd-gen-for-each-reg.h-usage.patch queue-5.10/x86-alternative-add-debug-prints-to-apply_retpolines.patch queue-5.10/objtool-extract-elf_symbol_add.patch queue-5.10/x86-use-return-thunk-in-asm-code.patch queue-5.10/objtool-remove-reloc-symbol-type-checks-in-get_alt_entry.patch queue-5.10/objtool-classify-symbols.patch queue-5.10/intel_idle-disable-ibrs-during-long-idle.patch queue-5.10/objtool-correctly-handle-retpoline-thunk-calls.patch queue-5.10/objtool-fix-.symtab_shndx-handling-for-elf_create_undef_symbol.patch queue-5.10/x86-retpoline-move-the-retpoline-thunk-declarations-to-nospec-branch.h.patch queue-5.10/objtool-support-asm-jump-tables.patch queue-5.10/x86-alternative-implement-.retpoline_sites-support.patch queue-5.10/objtool-x86-ignore-__x86_indirect_alt_-symbols.patch queue-5.10/objtool-fix-sls-validation-for-kcov-tail-call-replacement.patch queue-5.10/x86-alternative-try-inline-spectre_v2-retpoline-amd.patch queue-5.10/x86-entry-remove-skip_r11rcx.patch queue-5.10/objtool-explicitly-avoid-self-modifying-code-in-.altinstr_replacement.patch queue-5.10/x86-speculation-use-cached-host-spec_ctrl-value-for-guest-entry-exit.patch queue-5.10/x86-bugs-add-amd-retbleed-boot-parameter.patch queue-5.10/objtool-create-reloc-sections-implicitly.patch queue-5.10/x86-entry-add-kernel-ibrs-implementation.patch queue-5.10/objtool-treat-.text.__x86.-as-noinstr.patch queue-5.10/x86-lib-atomic64_386_32-rename-things.patch queue-5.10/objtool-introduce-cfi-hash.patch queue-5.10/objtool-default-ignore-int3-for-unreachable.patch queue-5.10/objtool-extract-elf_strtab_concat.patch queue-5.10/objtool-teach-get_alt_entry-about-more-relocation-types.patch queue-5.10/objtool-update-retpoline-validation.patch