On Mon, Apr 03, 2023 at 10:06:01PM +0800, Lai Jiangshan wrote: > diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile > index ca2fe186994b..7cc1254ca519 100644 > --- a/arch/x86/entry/Makefile > +++ b/arch/x86/entry/Makefile > @@ -8,11 +8,14 @@ UBSAN_SANITIZE := n > KCOV_INSTRUMENT := n > > CFLAGS_REMOVE_common.o = $(CC_FLAGS_FTRACE) > +CFLAGS_REMOVE_ist_entry.o = $(CC_FLAGS_FTRACE) $(RETHUNK_CFLAGS) This ^^^ > diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S > index 49ddc4dd3117..50a24cc83581 100644 > --- a/arch/x86/entry/entry_64.S > +++ b/arch/x86/entry/entry_64.S > @@ -443,6 +443,184 @@ SYM_CODE_END(\asmsym) > +.macro idtentry_ist vector asmsym cfunc user_cfunc has_error_code:req, stack_offset:req > +SYM_CODE_START(\asmsym) > + UNWIND_HINT_IRET_REGS offset=\has_error_code*8 > + ENDBR > + > + /* > + * Clear X86_EFLAGS_AC, X86_EFLAGS_DF and set a default ORIG_RAX. > + * > + * The code setting ORIG_RAX will not be replicated if interrupted. > + */ > + ASM_CLAC > + cld > + > + .if \has_error_code == 0 > + pushq $-1 /* ORIG_RAX: no syscall to restart */ > + .endif > + > + /* > + * No register can be touched except %rsp,%rflags,%rip before > + * pushing all the registers. It is indispensable for nested > + * atomic-IST-entry to replicate pushing the registers. > + */ > + PUSH_REGS > + > + /* > + * Finished pushing register, all registers can be touched by now. > + * > + * Clear registers for the C function ist_copy_regs_to_main_stack() > + * and the handler to avoid any possible exploitation of any > + * speculation attack. > + */ > + CLEAR_REGS > + > + /* > + * Copy the pt_regs to the IST main stack including the pt_regs of > + * the interrupted atomic-IST-entris, if any, by replicating. > + */ > + movq %rsp, %rdi /* pt_regs pointer on its own IST stack */ > + leaq PTREGS_SIZE-\stack_offset(%rsp), %rsi /* struct cea_exception_stacks pointer */ > + call ist_copy_regs_to_main_stack IIUC you do a CALL+RET here, before you call paranoid_entry ... > + > + /* > + * Commit stage. > + */ > +SYM_INNER_LABEL(start_commit_\asmsym, SYM_L_GLOBAL) > + /* > + * Switches to the IST main stack. Before the switching is done, > + * %rax is the copied pt_regs pointer in IST main stack. > + */ > + movq %rax, %rsp > + > + /* > + * The label should be immediate after the instruction that switches > + * the stack since there is code assuming there is only one single > + * instruction in the commit stage and the code assumes "%rsp in the > + * IST main stack is also the sign of ending a atomic-IST-entry". > + * (The code will be removed in future when %rip-based identifying > + * is added.) > + */ > +SYM_INNER_LABEL(commit_\asmsym, SYM_L_GLOBAL) > + > + /* > + * Now, it is on the IST main stack. For the whole kernel, the entries > + * of the IST exceptions can be seen from here because the inside > + * of the atomic-IST-entry can not be seen from the whole kernel > + * except in the atomic-IST-entry or #DF. > + */ > + UNWIND_HINT_REGS > + ENCODE_FRAME_POINTER > + > + /* > + * The code setting ORIG_RAX will not be replicated if interrupted. > + * So redo it here. > + */ > + .if \has_error_code == 0 > + movq $-1, ORIG_RAX(%rsp) /* ORIG_RAX: no syscall to restart */ > + .endif > + > + /* > + * If the entry is from userspace, switch stacks and treat it as > + * a normal entry. > + */ > + testb $3, CS(%rsp) > + jnz .Lfrom_usermode_switch_stack_\@ > + > + /* > + * paranoid_entry returns GS/CR3/SPEC_CTL information for > + * paranoid_exit in RBX/R14/R15. > + */ > + call paranoid_entry ... all the way down here, which will do: IBRS_ENTER; UNTRAIN_RET_FROM_CALL; Which thus breaks the whole RetBleed mess, since that must not do RET before that happens.
