On Wed 2018-02-07 08:00:11, Martin Schwidefsky wrote: > Add CONFIG_EXPOLINE to enable the use of the new -mindirect-branch= and > -mfunction_return= compiler options to create a kernel fortified against > the specte v2 attack. > > With CONFIG_EXPOLINE=y all indirect branches will be issued with an > execute type instruction. For z10 or newer the EXRL instruction will > be used, for older machines the EX instruction. The typical indirect > call > > basr %r14,%r1 > > is replaced with a PC relative call to a new thunk > > brasl %r14,__s390x_indirect_jump_r1 > > The thunk contains the EXRL/EX instruction to the indirect branch > > __s390x_indirect_jump_r1: > exrl 0,0f > j . > 0: br %r1 > > The detour via the execute type instruction has a performance impact. > To get rid of the detour the new kernel parameter "nospectre_v2" and > "spectre_v2=[on,off,auto]" can be used. If the parameter is specified > the kernel and module code will be patched at runtime. This is really unfortunate naming of kernel option. spectre_v2=off sounds like we are turning the "bug" off, but i somehow suspect you are turning the bug _workaround_ off. Pavel > Signed-off-by: Martin Schwidefsky <schwidefsky@xxxxxxxxxx> > --- > arch/s390/Kconfig | 28 +++++++++ > arch/s390/Makefile | 12 ++++ > arch/s390/include/asm/lowcore.h | 6 +- > arch/s390/include/asm/nospec-branch.h | 18 ++++++ > arch/s390/kernel/Makefile | 4 ++ > arch/s390/kernel/entry.S | 113 ++++++++++++++++++++++++++-------- > arch/s390/kernel/module.c | 62 ++++++++++++++++--- > arch/s390/kernel/nospec-branch.c | 100 ++++++++++++++++++++++++++++++ > arch/s390/kernel/setup.c | 4 ++ > arch/s390/kernel/smp.c | 1 + > arch/s390/kernel/vmlinux.lds.S | 14 +++++ > drivers/s390/char/Makefile | 2 + > 12 files changed, 329 insertions(+), 35 deletions(-) > create mode 100644 arch/s390/include/asm/nospec-branch.h > create mode 100644 arch/s390/kernel/nospec-branch.c > > diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig > index d514e25..d4a65bf 100644 > --- a/arch/s390/Kconfig > +++ b/arch/s390/Kconfig > @@ -557,6 +557,34 @@ config KERNEL_NOBP > > If unsure, say N. > > +config EXPOLINE > + def_bool n > + prompt "Avoid speculative indirect branches in the kernel" > + help > + Compile the kernel with the expoline compiler options to guard > + against kernel-to-user data leaks by avoiding speculative indirect > + branches. > + Requires a compiler with -mindirect-branch=thunk support for full > + protection. The kernel may run slower. > + > + If unsure, say N. > + > +choice > + prompt "Expoline default" > + depends on EXPOLINE > + default EXPOLINE_FULL > + > +config EXPOLINE_OFF > + bool "spectre_v2=off" > + > +config EXPOLINE_MEDIUM > + bool "spectre_v2=auto" > + > +config EXPOLINE_FULL > + bool "spectre_v2=on" > + > +endchoice > + > endmenu > > menu "Memory setup" > diff --git a/arch/s390/Makefile b/arch/s390/Makefile > index fd691c4..2f925ef 100644 > --- a/arch/s390/Makefile > +++ b/arch/s390/Makefile > @@ -78,6 +78,18 @@ ifeq ($(call cc-option-yn,-mwarn-dynamicstack),y) > cflags-$(CONFIG_WARN_DYNAMIC_STACK) += -mwarn-dynamicstack > endif > > +ifdef CONFIG_EXPOLINE > + ifeq ($(call cc-option-yn,$(CC_FLAGS_MARCH) -mindirect-branch=thunk),y) > + CC_FLAGS_EXPOLINE := -mindirect-branch=thunk > + CC_FLAGS_EXPOLINE += -mfunction-return=thunk > + CC_FLAGS_EXPOLINE += -mindirect-branch-table > + export CC_FLAGS_EXPOLINE > + cflags-y += $(CC_FLAGS_EXPOLINE) > + else > + $(warning "Your gcc lacks the -mindirect-branch= option") > + endif > +endif > + > ifdef CONFIG_FUNCTION_TRACER > # make use of hotpatch feature if the compiler supports it > cc_hotpatch := -mhotpatch=0,3 > diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h > index c63986a..5bc8888 100644 > --- a/arch/s390/include/asm/lowcore.h > +++ b/arch/s390/include/asm/lowcore.h > @@ -136,7 +136,11 @@ struct lowcore { > __u64 vdso_per_cpu_data; /* 0x03b8 */ > __u64 machine_flags; /* 0x03c0 */ > __u64 gmap; /* 0x03c8 */ > - __u8 pad_0x03d0[0x0e00-0x03d0]; /* 0x03d0 */ > + __u8 pad_0x03d0[0x0400-0x03d0]; /* 0x03d0 */ > + > + /* br %r1 trampoline */ > + __u16 br_r1_trampoline; /* 0x0400 */ > + __u8 pad_0x0402[0x0e00-0x0402]; /* 0x0402 */ > > /* > * 0xe00 contains the address of the IPL Parameter Information > diff --git a/arch/s390/include/asm/nospec-branch.h b/arch/s390/include/asm/nospec-branch.h > new file mode 100644 > index 0000000..7df48e5 > --- /dev/null > +++ b/arch/s390/include/asm/nospec-branch.h > @@ -0,0 +1,18 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +#ifndef _ASM_S390_EXPOLINE_H > +#define _ASM_S390_EXPOLINE_H > + > +#ifndef __ASSEMBLY__ > + > +#include <linux/types.h> > + > +extern int nospec_call_disable; > +extern int nospec_return_disable; > + > +void nospec_init_branches(void); > +void nospec_call_revert(s32 *start, s32 *end); > +void nospec_return_revert(s32 *start, s32 *end); > + > +#endif /* __ASSEMBLY__ */ > + > +#endif /* _ASM_S390_EXPOLINE_H */ > diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile > index 909bce6..7f27e3d 100644 > --- a/arch/s390/kernel/Makefile > +++ b/arch/s390/kernel/Makefile > @@ -29,6 +29,7 @@ UBSAN_SANITIZE_early.o := n > # > ifneq ($(CC_FLAGS_MARCH),-march=z900) > CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH) > +CFLAGS_REMOVE_als.o += $(CC_FLAGS_EXPOLINE) > CFLAGS_als.o += -march=z900 > AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH) > AFLAGS_head.o += -march=z900 > @@ -63,6 +64,9 @@ obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o > > extra-y += head.o head64.o vmlinux.lds > > +obj-$(CONFIG_EXPOLINE) += nospec-branch.o > +CFLAGS_REMOVE_expoline.o += $(CC_FLAGS_EXPOLINE) > + > obj-$(CONFIG_MODULES) += module.o > obj-$(CONFIG_SMP) += smp.o > obj-$(CONFIG_SCHED_TOPOLOGY) += topology.o > diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S > index 53145b5..13a133a 100644 > --- a/arch/s390/kernel/entry.S > +++ b/arch/s390/kernel/entry.S > @@ -222,6 +222,68 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART) > .popsection > .endm > > +#ifdef CONFIG_EXPOLINE > + > + .macro GEN_BR_THUNK name,reg,tmp > + .section .text.\name,"axG",@progbits,\name,comdat > + .globl \name > + .hidden \name > + .type \name,@function > +\name: > + .cfi_startproc > +#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES > + exrl 0,0f > +#else > + larl \tmp,0f > + ex 0,0(\tmp) > +#endif > + j . > +0: br \reg > + .cfi_endproc > + .endm > + > + GEN_BR_THUNK __s390x_indirect_jump_r1use_r9,%r9,%r1 > + GEN_BR_THUNK __s390x_indirect_jump_r1use_r14,%r14,%r1 > + GEN_BR_THUNK __s390x_indirect_jump_r11use_r14,%r14,%r11 > + > + .macro BASR_R14_R9 > +0: brasl %r14,__s390x_indirect_jump_r1use_r9 > + .pushsection .s390_indirect_branches,"a",@progbits > + .long 0b-. > + .popsection > + .endm > + > + .macro BR_R1USE_R14 > +0: jg __s390x_indirect_jump_r1use_r14 > + .pushsection .s390_indirect_branches,"a",@progbits > + .long 0b-. > + .popsection > + .endm > + > + .macro BR_R11USE_R14 > +0: jg __s390x_indirect_jump_r11use_r14 > + .pushsection .s390_indirect_branches,"a",@progbits > + .long 0b-. > + .popsection > + .endm > + > +#else /* CONFIG_EXPOLINE */ > + > + .macro BASR_R14_R9 > + basr %r14,%r9 > + .endm > + > + .macro BR_R1USE_R14 > + br %r14 > + .endm > + > + .macro BR_R11USE_R14 > + br %r14 > + .endm > + > +#endif /* CONFIG_EXPOLINE */ > + > + > .section .kprobes.text, "ax" > .Ldummy: > /* > @@ -237,7 +299,7 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART) > ENTRY(__bpon) > .globl __bpon > BPON > - br %r14 > + BR_R1USE_R14 > > /* > * Scheduler resume function, called by switch_to > @@ -261,9 +323,9 @@ ENTRY(__switch_to) > mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next > lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task > TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPP > - bzr %r14 > + jz 0f > .insn s,0xb2800000,__LC_LPP # set program parameter > - br %r14 > +0: BR_R1USE_R14 > > .L__critical_start: > > @@ -330,7 +392,7 @@ sie_exit: > xgr %r5,%r5 > lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers > lg %r2,__SF_EMPTY+16(%r15) # return exit reason code > - br %r14 > + BR_R1USE_R14 > .Lsie_fault: > lghi %r14,-EFAULT > stg %r14,__SF_EMPTY+16(%r15) # set exit reason code > @@ -389,7 +451,7 @@ ENTRY(system_call) > lgf %r9,0(%r8,%r10) # get system call add. > TSTMSK __TI_flags(%r12),_TIF_TRACE > jnz .Lsysc_tracesys > - basr %r14,%r9 # call sys_xxxx > + BASR_R14_R9 # call sys_xxxx > stg %r2,__PT_R2(%r11) # store return value > > .Lsysc_return: > @@ -574,7 +636,7 @@ ENTRY(system_call) > lmg %r3,%r7,__PT_R3(%r11) > stg %r7,STACK_FRAME_OVERHEAD(%r15) > lg %r2,__PT_ORIG_GPR2(%r11) > - basr %r14,%r9 # call sys_xxx > + BASR_R14_R9 # call sys_xxx > stg %r2,__PT_R2(%r11) # store return value > .Lsysc_tracenogo: > TSTMSK __TI_flags(%r12),_TIF_TRACE > @@ -598,7 +660,7 @@ ENTRY(ret_from_fork) > lmg %r9,%r10,__PT_R9(%r11) # load gprs > ENTRY(kernel_thread_starter) > la %r2,0(%r10) > - basr %r14,%r9 > + BASR_R14_R9 > j .Lsysc_tracenogo > > /* > @@ -678,9 +740,9 @@ ENTRY(pgm_check_handler) > nill %r10,0x007f > sll %r10,2 > je .Lpgm_return > - lgf %r1,0(%r10,%r1) # load address of handler routine > + lgf %r9,0(%r10,%r1) # load address of handler routine > lgr %r2,%r11 # pass pointer to pt_regs > - basr %r14,%r1 # branch to interrupt-handler > + BASR_R14_R9 # branch to interrupt-handler > .Lpgm_return: > LOCKDEP_SYS_EXIT > tm __PT_PSW+1(%r11),0x01 # returning to user ? > @@ -998,7 +1060,7 @@ ENTRY(psw_idle) > stpt __TIMER_IDLE_ENTER(%r2) > .Lpsw_idle_lpsw: > lpswe __SF_EMPTY(%r15) > - br %r14 > + BR_R1USE_R14 > .Lpsw_idle_end: > > /* > @@ -1012,7 +1074,7 @@ ENTRY(save_fpu_regs) > lg %r2,__LC_CURRENT > aghi %r2,__TASK_thread > TSTMSK __LC_CPU_FLAGS,_CIF_FPU > - bor %r14 > + jo .Lsave_fpu_regs_exit > stfpc __THREAD_FPU_fpc(%r2) > lg %r3,__THREAD_FPU_regs(%r2) > TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX > @@ -1039,7 +1101,8 @@ ENTRY(save_fpu_regs) > std 15,120(%r3) > .Lsave_fpu_regs_done: > oi __LC_CPU_FLAGS+7,_CIF_FPU > - br %r14 > +.Lsave_fpu_regs_exit: > + BR_R1USE_R14 > .Lsave_fpu_regs_end: > EXPORT_SYMBOL(save_fpu_regs) > > @@ -1057,7 +1120,7 @@ load_fpu_regs: > lg %r4,__LC_CURRENT > aghi %r4,__TASK_thread > TSTMSK __LC_CPU_FLAGS,_CIF_FPU > - bnor %r14 > + jno .Lload_fpu_regs_exit > lfpc __THREAD_FPU_fpc(%r4) > TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX > lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area > @@ -1084,7 +1147,8 @@ load_fpu_regs: > ld 15,120(%r4) > .Lload_fpu_regs_done: > ni __LC_CPU_FLAGS+7,255-_CIF_FPU > - br %r14 > +.Lload_fpu_regs_exit: > + BR_R1USE_R14 > .Lload_fpu_regs_end: > > .L__critical_end: > @@ -1301,7 +1365,7 @@ cleanup_critical: > jl 0f > clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end > jl .Lcleanup_load_fpu_regs > -0: br %r14 > +0: BR_R11USE_R14 > > .align 8 > .Lcleanup_table: > @@ -1337,7 +1401,7 @@ cleanup_critical: > ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE > lctlg %c1,%c1,__LC_USER_ASCE # load primary asce > larl %r9,sie_exit # skip forward to sie_exit > - br %r14 > + BR_R11USE_R14 > #endif > > .Lcleanup_system_call: > @@ -1390,7 +1454,7 @@ cleanup_critical: > stg %r15,56(%r11) # r15 stack pointer > # set new psw address and exit > larl %r9,.Lsysc_do_svc > - br %r14 > + BR_R11USE_R14 > .Lcleanup_system_call_insn: > .quad system_call > .quad .Lsysc_stmg > @@ -1402,7 +1466,7 @@ cleanup_critical: > > .Lcleanup_sysc_tif: > larl %r9,.Lsysc_tif > - br %r14 > + BR_R11USE_R14 > > .Lcleanup_sysc_restore: > # check if stpt has been executed > @@ -1419,14 +1483,14 @@ cleanup_critical: > mvc 0(64,%r11),__PT_R8(%r9) > lmg %r0,%r7,__PT_R0(%r9) > 1: lmg %r8,%r9,__LC_RETURN_PSW > - br %r14 > + BR_R11USE_R14 > .Lcleanup_sysc_restore_insn: > .quad .Lsysc_exit_timer > .quad .Lsysc_done - 4 > > .Lcleanup_io_tif: > larl %r9,.Lio_tif > - br %r14 > + BR_R11USE_R14 > > .Lcleanup_io_restore: > # check if stpt has been executed > @@ -1440,7 +1504,7 @@ cleanup_critical: > mvc 0(64,%r11),__PT_R8(%r9) > lmg %r0,%r7,__PT_R0(%r9) > 1: lmg %r8,%r9,__LC_RETURN_PSW > - br %r14 > + BR_R11USE_R14 > .Lcleanup_io_restore_insn: > .quad .Lio_exit_timer > .quad .Lio_done - 4 > @@ -1493,17 +1557,17 @@ cleanup_critical: > # prepare return psw > nihh %r8,0xfcfd # clear irq & wait state bits > lg %r9,48(%r11) # return from psw_idle > - br %r14 > + BR_R11USE_R14 > .Lcleanup_idle_insn: > .quad .Lpsw_idle_lpsw > > .Lcleanup_save_fpu_regs: > larl %r9,save_fpu_regs > - br %r14 > + BR_R11USE_R14 > > .Lcleanup_load_fpu_regs: > larl %r9,load_fpu_regs > - br %r14 > + BR_R11USE_R14 > > /* > * Integer constants > @@ -1523,7 +1587,6 @@ cleanup_critical: > .Lsie_crit_mcck_length: > .quad .Lsie_skip - .Lsie_entry > #endif > - > .section .rodata, "a" > #define SYSCALL(esame,emu) .long esame > .globl sys_call_table > diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c > index b7abfad..1fc6d1f 100644 > --- a/arch/s390/kernel/module.c > +++ b/arch/s390/kernel/module.c > @@ -19,6 +19,8 @@ > #include <linux/moduleloader.h> > #include <linux/bug.h> > #include <asm/alternative.h> > +#include <asm/nospec-branch.h> > +#include <asm/facility.h> > > #if 0 > #define DEBUGP printk > @@ -156,7 +158,11 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, > me->arch.got_offset = me->core_layout.size; > me->core_layout.size += me->arch.got_size; > me->arch.plt_offset = me->core_layout.size; > - me->core_layout.size += me->arch.plt_size; > + if (me->arch.plt_size) { > + if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_call_disable) > + me->arch.plt_size += PLT_ENTRY_SIZE; > + me->core_layout.size += me->arch.plt_size; > + } > return 0; > } > > @@ -310,9 +316,21 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, > unsigned int *ip; > ip = me->core_layout.base + me->arch.plt_offset + > info->plt_offset; > - ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */ > - ip[1] = 0x100a0004; > - ip[2] = 0x07f10000; > + ip[0] = 0x0d10e310; /* basr 1,0 */ > + ip[1] = 0x100a0004; /* lg 1,10(1) */ > + if (IS_ENABLED(CONFIG_EXPOLINE) && > + !nospec_call_disable) { > + unsigned int *ij; > + ij = me->core_layout.base + > + me->arch.plt_offset + > + me->arch.plt_size - PLT_ENTRY_SIZE; > + ip[2] = 0xa7f40000 + /* j __jump_r1 */ > + (unsigned int)(u16) > + (((unsigned long) ij - 8 - > + (unsigned long) ip) / 2); > + } else { > + ip[2] = 0x07f10000; /* br %r1 */ > + } > ip[3] = (unsigned int) (val >> 32); > ip[4] = (unsigned int) val; > info->plt_initialized = 1; > @@ -418,16 +436,42 @@ int module_finalize(const Elf_Ehdr *hdr, > struct module *me) > { > const Elf_Shdr *s; > - char *secstrings; > + char *secstrings, *secname; > + void *aseg; > + > + if (IS_ENABLED(CONFIG_EXPOLINE) && > + !nospec_call_disable && me->arch.plt_size) { > + unsigned int *ij; > + > + ij = me->core_layout.base + me->arch.plt_offset + > + me->arch.plt_size - PLT_ENTRY_SIZE; > + if (test_facility(35)) { > + ij[0] = 0xc6000000; /* exrl %r0,.+10 */ > + ij[1] = 0x0005a7f4; /* j . */ > + ij[2] = 0x000007f1; /* br %r1 */ > + } else { > + ij[0] = 0x44000000 | (unsigned int) > + offsetof(struct lowcore, br_r1_trampoline); > + ij[1] = 0xa7f40000; /* j . */ > + } > + } > > secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; > for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { > - if (!strcmp(".altinstructions", secstrings + s->sh_name)) { > - /* patch .altinstructions */ > - void *aseg = (void *)s->sh_addr; > + aseg = (void *) s->sh_addr; > + secname = secstrings + s->sh_name; > > + if (!strcmp(".altinstructions", secname)) > + /* patch .altinstructions */ > apply_alternatives(aseg, aseg + s->sh_size); > - } > + > + if (IS_ENABLED(CONFIG_EXPOLINE) && > + (!strcmp(".nospec_call_table", secname))) > + nospec_call_revert(aseg, aseg + s->sh_size); > + > + if (IS_ENABLED(CONFIG_EXPOLINE) && > + (!strcmp(".nospec_return_table", secname))) > + nospec_return_revert(aseg, aseg + s->sh_size); > } > > jump_label_apply_nops(me); > diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c > new file mode 100644 > index 0000000..69d7fcf > --- /dev/null > +++ b/arch/s390/kernel/nospec-branch.c > @@ -0,0 +1,100 @@ > +// SPDX-License-Identifier: GPL-2.0 > +#include <linux/module.h> > +#include <asm/nospec-branch.h> > + > +int nospec_call_disable = IS_ENABLED(EXPOLINE_OFF); > +int nospec_return_disable = !IS_ENABLED(EXPOLINE_FULL); > + > +static int __init nospectre_v2_setup_early(char *str) > +{ > + nospec_call_disable = 1; > + nospec_return_disable = 1; > + return 0; > +} > +early_param("nospectre_v2", nospectre_v2_setup_early); > + > +static int __init spectre_v2_setup_early(char *str) > +{ > + if (str && !strncmp(str, "on", 2)) { > + nospec_call_disable = 0; > + nospec_return_disable = 0; > + } > + if (str && !strncmp(str, "off", 3)) { > + nospec_call_disable = 1; > + nospec_return_disable = 1; > + } > + if (str && !strncmp(str, "auto", 4)) { > + nospec_call_disable = 0; > + nospec_return_disable = 1; > + } > + return 0; > +} > +early_param("spectre_v2", spectre_v2_setup_early); > + > +static void __init_or_module __nospec_revert(s32 *start, s32 *end) > +{ > + enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type; > + u8 *instr, *thunk, *br; > + u8 insnbuf[6]; > + s32 *epo; > + > + /* Second part of the instruction replace is always a nop */ > + memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x00, 0x00 }, 4); > + for (epo = start; epo < end; epo++) { > + instr = (u8 *) epo + *epo; > + if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04) > + type = BRCL_EXPOLINE; /* brcl instruction */ > + else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05) > + type = BRASL_EXPOLINE; /* brasl instruction */ > + else > + continue; > + thunk = instr + (*(int *)(instr + 2)) * 2; > + if (thunk[0] == 0xc6 && thunk[1] == 0x00) > + /* exrl %r0,<target-br> */ > + br = thunk + (*(int *)(thunk + 2)) * 2; > + else if (thunk[0] == 0xc0 && (thunk[1] & 0x0f) == 0x00 && > + thunk[6] == 0x44 && thunk[7] == 0x00 && > + (thunk[8] & 0x0f) == 0x00 && thunk[9] == 0x00 && > + (thunk[1] & 0xf0) == (thunk[8] & 0xf0)) > + /* larl %rx,<target br> + ex %r0,0(%rx) */ > + br = thunk + (*(int *)(thunk + 2)) * 2; > + else > + continue; > + if (br[0] != 0x07 || (br[1] & 0xf0) != 0xf0) > + continue; > + switch (type) { > + case BRCL_EXPOLINE: > + /* brcl to thunk, replace with br + nop */ > + insnbuf[0] = br[0]; > + insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); > + break; > + case BRASL_EXPOLINE: > + /* brasl to thunk, replace with basr + nop */ > + insnbuf[0] = 0x0d; > + insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f); > + break; > + } > + > + s390_kernel_write(instr, insnbuf, 6); > + } > +} > + > +void __init_or_module nospec_call_revert(s32 *start, s32 *end) > +{ > + if (nospec_call_disable) > + __nospec_revert(start, end); > +} > + > +void __init_or_module nospec_return_revert(s32 *start, s32 *end) > +{ > + if (nospec_return_disable) > + __nospec_revert(start, end); > +} > + > +extern s32 __nospec_call_start[], __nospec_call_end[]; > +extern s32 __nospec_return_start[], __nospec_return_end[]; > +void __init nospec_init_branches(void) > +{ > + nospec_call_revert(__nospec_call_start, __nospec_call_end); > + nospec_return_revert(__nospec_return_start, __nospec_return_end); > +} > diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c > index bcd2a4a..a6a91f0 100644 > --- a/arch/s390/kernel/setup.c > +++ b/arch/s390/kernel/setup.c > @@ -68,6 +68,7 @@ > #include <asm/sysinfo.h> > #include <asm/numa.h> > #include <asm/alternative.h> > +#include <asm/nospec-branch.h> > #include "entry.h" > > /* > @@ -379,6 +380,7 @@ static void __init setup_lowcore(void) > lc->spinlock_index = 0; > arch_spin_lock_setup(0); > #endif > + lc->br_r1_trampoline = 0x07f1; /* br %r1 */ > > set_prefix((u32)(unsigned long) lc); > lowcore_ptr[0] = lc; > @@ -954,6 +956,8 @@ void __init setup_arch(char **cmdline_p) > set_preferred_console(); > > apply_alternative_instructions(); > + if (IS_ENABLED(CONFIG_EXPOLINE)) > + nospec_init_branches(); > > /* Setup zfcpdump support */ > setup_zfcpdump(); > diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c > index 2fd7d60..a4a9fe1 100644 > --- a/arch/s390/kernel/smp.c > +++ b/arch/s390/kernel/smp.c > @@ -214,6 +214,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) > lc->cpu_nr = cpu; > lc->spinlock_lockval = arch_spin_lockval(cpu); > lc->spinlock_index = 0; > + lc->br_r1_trampoline = 0x07f1; /* br %r1 */ > if (nmi_alloc_per_cpu(lc)) > goto out; > if (vdso_alloc_per_cpu(lc)) > diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S > index 608cf29..08d12cf 100644 > --- a/arch/s390/kernel/vmlinux.lds.S > +++ b/arch/s390/kernel/vmlinux.lds.S > @@ -123,6 +123,20 @@ SECTIONS > *(.altinstr_replacement) > } > > + /* > + * Table with the patch locations to undo expolines > + */ > + .nospec_call_table : { > + __nospec_call_start = . ; > + *(.s390_indirect*) > + __nospec_call_end = . ; > + } > + .nospec_return_table : { > + __nospec_return_start = . ; > + *(.s390_return*) > + __nospec_return_end = . ; > + } > + > /* early.c uses stsi, which requires page aligned data. */ > . = ALIGN(PAGE_SIZE); > INIT_DATA_SECTION(0x100) > diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile > index 614b44e..a2b33a2 100644 > --- a/drivers/s390/char/Makefile > +++ b/drivers/s390/char/Makefile > @@ -19,6 +19,8 @@ endif > > CFLAGS_sclp_early_core.o += -D__NO_FORTIFY > > +CFLAGS_REMOVE_sclp_early_core.o += $(CC_FLAGS_EXPOLINE) > + > obj-y += ctrlchar.o keyboard.o defkeymap.o sclp.o sclp_rw.o sclp_quiesce.o \ > sclp_cmd.o sclp_config.o sclp_cpi_sys.o sclp_ocf.o sclp_ctl.o \ > sclp_early.o sclp_early_core.o -- (english) http://www.livejournal.com/~pavelmachek (cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
Attachment:
signature.asc
Description: Digital signature