Add assembly helpers to entry_64.S that invoke hooks from kmsan_entry.c and notify KMSAN about interrupts. Also call these hooks from kernel/softirq.c This is needed to switch between several KMSAN contexts holding function parameter metadata. Signed-off-by: Alexander Potapenko <glider@xxxxxxxxxx> To: Alexander Potapenko <glider@xxxxxxxxxx> Cc: Jens Axboe <axboe@xxxxxxxxx> Cc: Andy Lutomirski <luto@xxxxxxxxxx> Cc: Vegard Nossum <vegard.nossum@xxxxxxxxxx> Cc: Dmitry Vyukov <dvyukov@xxxxxxxxxx> Cc: Marco Elver <elver@xxxxxxxxxx> Cc: Andrey Konovalov <andreyknvl@xxxxxxxxxx> Cc: Christoph Hellwig <hch@xxxxxx> Cc: linux-mm@xxxxxxxxx --- v4: - moved softirq changes to this patch Change-Id: I3037d51672fe69d09e588b27adb2d9fdc6ad3a7d --- arch/x86/entry/entry_64.S | 16 ++++++++++++++++ kernel/softirq.c | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 0e9504fabe526..03f5a32b0af4d 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -35,6 +35,7 @@ #include <asm/asm.h> #include <asm/smap.h> #include <asm/pgtable_types.h> +#include <asm/kmsan.h> #include <asm/export.h> #include <asm/frame.h> #include <asm/nospec-branch.h> @@ -575,6 +576,7 @@ SYM_CODE_START(interrupt_entry) 1: ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 + KMSAN_INTERRUPT_ENTER /* We entered an interrupt context - irqs are off: */ TRACE_IRQS_OFF @@ -604,12 +606,14 @@ SYM_CODE_START_LOCAL(common_interrupt) addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ call interrupt_entry UNWIND_HINT_REGS indirect=1 + KMSAN_UNPOISON_PT_REGS call do_IRQ /* rdi points to pt_regs */ /* 0(%rsp): old RSP */ ret_from_intr: DISABLE_INTERRUPTS(CLBR_ANY) TRACE_IRQS_OFF + KMSAN_INTERRUPT_EXIT LEAVE_IRQ_STACK testb $3, CS(%rsp) @@ -801,6 +805,7 @@ SYM_CODE_START(\sym) .Lcommon_\sym: call interrupt_entry UNWIND_HINT_REGS indirect=1 + KMSAN_UNPOISON_PT_REGS call \do_sym /* rdi points to pt_regs */ jmp ret_from_intr SYM_CODE_END(\sym) @@ -908,15 +913,18 @@ apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt .if \shift_ist != -1 subq $\ist_offset, CPU_TSS_IST(\shift_ist) + KMSAN_IST_ENTER(\shift_ist) .endif .if \read_cr2 movq %r12, %rdx /* Move CR2 into 3rd argument */ .endif + KMSAN_UNPOISON_PT_REGS call \do_sym .if \shift_ist != -1 + KMSAN_IST_EXIT(\shift_ist) addq $\ist_offset, CPU_TSS_IST(\shift_ist) .endif @@ -1079,7 +1087,9 @@ SYM_FUNC_START(do_softirq_own_stack) pushq %rbp mov %rsp, %rbp ENTER_IRQ_STACK regs=0 old_rsp=%r11 + KMSAN_SOFTIRQ_ENTER call __do_softirq + KMSAN_SOFTIRQ_EXIT LEAVE_IRQ_STACK regs=0 leaveq ret @@ -1466,9 +1476,12 @@ SYM_CODE_START(nmi) * done with the NMI stack. */ + KMSAN_NMI_ENTER movq %rsp, %rdi movq $-1, %rsi + KMSAN_UNPOISON_PT_REGS call do_nmi + KMSAN_NMI_EXIT /* * Return back to user mode. We must *not* do the normal exit @@ -1678,10 +1691,13 @@ end_repeat_nmi: call paranoid_entry UNWIND_HINT_REGS + KMSAN_NMI_ENTER /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ movq %rsp, %rdi movq $-1, %rsi + KMSAN_UNPOISON_PT_REGS call do_nmi + KMSAN_NMI_EXIT /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 diff --git a/kernel/softirq.c b/kernel/softirq.c index 0427a86743a46..98c5f4062cbfe 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -11,6 +11,7 @@ #include <linux/export.h> #include <linux/kernel_stat.h> +#include <linux/kmsan.h> #include <linux/interrupt.h> #include <linux/init.h> #include <linux/mm.h> @@ -370,7 +371,9 @@ static inline void invoke_softirq(void) * it is the irq stack, because it should be near empty * at this stage. */ + kmsan_context_enter(); __do_softirq(); + kmsan_context_exit(); #else /* * Otherwise, irq_exit() is called on the task stack that can @@ -600,7 +603,9 @@ static void run_ksoftirqd(unsigned int cpu) * We can safely run softirq on inline stack, as we are not deep * in the task stack here. */ + kmsan_context_enter(); __do_softirq(); + kmsan_context_exit(); local_irq_enable(); cond_resched(); return; -- 2.25.1.696.g5e7596f4ac-goog
