Re: [PATCH v4 7/7] ARM: implement support for vmap'ed stacks

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Hi,

On 22.11.2021 10:28, Ard Biesheuvel wrote:
> Wire up the generic support for managing task stack allocations via vmalloc,
> and implement the entry code that detects whether we faulted because of a
> stack overrun (or future stack overrun caused by pushing the pt_regs array)
>
> While this adds a fair amount of tricky entry asm code, it should be
> noted that it only adds a TST + branch to the svc_entry path. The code
> implementing the non-trivial handling of the overflow stack is emitted
> out-of-line into the .text section.
>
> Since on ARM, we rely on do_translation_fault() to keep PMD level page
> table entries that cover the vmalloc region up to date, we need to
> ensure that we don't hit such a stale PMD entry when accessing the
> stack. So we do a dummy read from the new stack while still running from
> the old one on the context switch path, and bump the vmalloc_seq counter
> when PMD level entries in the vmalloc range are modified, so that the MM
> switch fetches the latest version of the entries.
>
> Note that we need to increase the per-mode stack by 1 word, to gain some
> space to stash a GPR until we know it is safe to touch the stack.
> However, due to the cacheline alignment of the struct, this does not
> actually increase the memory footprint of the struct stack array at all.
>
> Signed-off-by: Ard Biesheuvel <ardb@xxxxxxxxxx>
> Tested-by: Keith Packard <keithpac@xxxxxxxxxx>


This patch landed recently in linux-next 20211220 as commit a1c510d0adc6 
("ARM: implement support for vmap'ed stacks"). Sadly it breaks 
suspend/resume operation on all ARM 32bit Exynos SoCs. Probably the 
suspend/resume related code must be updated somehow (it partially works 
on physical addresses and disabled MMU), but I didn't analyze it yet. If 
you have any hints, let me know.


> ---
>   arch/arm/Kconfig                   |  1 +
>   arch/arm/include/asm/page.h        |  4 +
>   arch/arm/include/asm/thread_info.h |  8 ++
>   arch/arm/kernel/entry-armv.S       | 97 +++++++++++++++++++-
>   arch/arm/kernel/entry-header.S     | 37 ++++++++
>   arch/arm/kernel/irq.c              |  9 +-
>   arch/arm/kernel/setup.c            |  8 +-
>   arch/arm/kernel/sleep.S            |  8 ++
>   arch/arm/kernel/traps.c            | 80 +++++++++++++++-
>   arch/arm/kernel/unwind.c           |  3 +-
>   arch/arm/kernel/vmlinux.lds.S      |  4 +-
>   11 files changed, 244 insertions(+), 15 deletions(-)
>
> diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
> index b1eba1b4168c..7a0853bd298f 100644
> --- a/arch/arm/Kconfig
> +++ b/arch/arm/Kconfig
> @@ -127,6 +127,7 @@ config ARM
>   	select RTC_LIB
>   	select SYS_SUPPORTS_APM_EMULATION
>   	select THREAD_INFO_IN_TASK if CURRENT_POINTER_IN_TPIDRURO
> +	select HAVE_ARCH_VMAP_STACK if THREAD_INFO_IN_TASK && (!LD_IS_LLD || LLD_VERSION >= 140000)
>   	select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M
>   	# Above selects are sorted alphabetically; please add new ones
>   	# according to that.  Thanks.
> diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
> index 11b058a72a5b..7b871ed99ccf 100644
> --- a/arch/arm/include/asm/page.h
> +++ b/arch/arm/include/asm/page.h
> @@ -149,6 +149,10 @@ extern void copy_page(void *to, const void *from);
>   #include <asm/pgtable-2level-types.h>
>   #endif
>   
> +#ifdef CONFIG_VMAP_STACK
> +#define ARCH_PAGE_TABLE_SYNC_MASK	PGTBL_PMD_MODIFIED
> +#endif
> +
>   #endif /* CONFIG_MMU */
>   
>   typedef struct page *pgtable_t;
> diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
> index 164e15f26485..004b89d86224 100644
> --- a/arch/arm/include/asm/thread_info.h
> +++ b/arch/arm/include/asm/thread_info.h
> @@ -25,6 +25,14 @@
>   #define THREAD_SIZE		(PAGE_SIZE << THREAD_SIZE_ORDER)
>   #define THREAD_START_SP		(THREAD_SIZE - 8)
>   
> +#ifdef CONFIG_VMAP_STACK
> +#define THREAD_ALIGN		(2 * THREAD_SIZE)
> +#else
> +#define THREAD_ALIGN		THREAD_SIZE
> +#endif
> +
> +#define OVERFLOW_STACK_SIZE	SZ_4K
> +
>   #ifndef __ASSEMBLY__
>   
>   struct task_struct;
> diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
> index b447f7d0708c..54210dce80e1 100644
> --- a/arch/arm/kernel/entry-armv.S
> +++ b/arch/arm/kernel/entry-armv.S
> @@ -57,6 +57,10 @@ UNWIND(	.setfp	fpreg, sp		)
>   	@
>   	subs	r2, sp, r0		@ SP above bottom of IRQ stack?
>   	rsbscs	r2, r2, #THREAD_SIZE	@ ... and below the top?
> +#ifdef CONFIG_VMAP_STACK
> +	ldr_l	r2, high_memory, cc	@ End of the linear region
> +	cmpcc	r2, r0			@ Stack pointer was below it?
> +#endif
>   	movcs	sp, r0			@ If so, revert to incoming SP
>   
>   #ifndef CONFIG_UNWINDER_ARM
> @@ -188,13 +192,18 @@ ENDPROC(__und_invalid)
>   #define SPFIX(code...)
>   #endif
>   
> -	.macro	svc_entry, stack_hole=0, trace=1, uaccess=1
> +	.macro	svc_entry, stack_hole=0, trace=1, uaccess=1, overflow_check=1
>    UNWIND(.fnstart		)
> - UNWIND(.save {r0 - pc}		)
>   	sub	sp, sp, #(SVC_REGS_SIZE + \stack_hole)
> + THUMB(	add	sp, r1		)	@ get SP in a GPR without
> + THUMB(	sub	r1, sp, r1	)	@ using a temp register
> +
> +	.if	\overflow_check
> + UNWIND(.save	{r0 - pc}	)
> +	do_overflow_check (SVC_REGS_SIZE + \stack_hole)
> +	.endif
> +
>   #ifdef CONFIG_THUMB2_KERNEL
> -	add	sp, r1			@ get SP in a GPR without
> -	sub	r1, sp, r1		@ using a temp register
>   	tst	r1, #4			@ test stack pointer alignment
>   	sub	r1, sp, r1		@ restore original R0
>   	sub	sp, r1			@ restore original SP
> @@ -827,12 +836,20 @@ ENTRY(__switch_to)
>   	str	r7, [r8]
>   #endif
>   	mov	r0, r5
> -#if !defined(CONFIG_THUMB2_KERNEL)
> +#if !defined(CONFIG_THUMB2_KERNEL) && !defined(CONFIG_VMAP_STACK)
>   	set_current r7
>   	ldmia	r4, {r4 - sl, fp, sp, pc}	@ Load all regs saved previously
>   #else
>   	mov	r1, r7
>   	ldmia	r4, {r4 - sl, fp, ip, lr}	@ Load all regs saved previously
> +#ifdef CONFIG_VMAP_STACK
> +	@
> +	@ Do a dummy read from the new stack while running from the old one so
> +	@ that we can rely on do_translation_fault() to fix up any stale PMD
> +	@ entries covering the vmalloc region.
> +	@
> +	ldr	r2, [ip]
> +#endif
>   
>   	@ When CONFIG_THREAD_INFO_IN_TASK=n, the update of SP itself is what
>   	@ effectuates the task switch, as that is what causes the observable
> @@ -849,6 +866,76 @@ ENTRY(__switch_to)
>    UNWIND(.fnend		)
>   ENDPROC(__switch_to)
>   
> +#ifdef CONFIG_VMAP_STACK
> +	.text
> +	.align	2
> +__bad_stack:
> +	@
> +	@ We've just detected an overflow. We need to load the address of this
> +	@ CPU's overflow stack into the stack pointer register. We have only one
> +	@ scratch register so let's use a sequence of ADDs including one
> +	@ involving the PC, and decorate them with PC-relative group
> +	@ relocations. As these are ARM only, switch to ARM mode first.
> +	@
> +	@ We enter here with IP clobbered and its value stashed on the mode
> +	@ stack.
> +	@
> +THUMB(	bx	pc		)
> +THUMB(	nop			)
> +THUMB(	.arm			)
> +	mrc	p15, 0, ip, c13, c0, 4		@ Get per-CPU offset
> +
> +	.globl	overflow_stack_ptr
> +	.reloc	0f, R_ARM_ALU_PC_G0_NC, overflow_stack_ptr
> +	.reloc	1f, R_ARM_ALU_PC_G1_NC, overflow_stack_ptr
> +	.reloc	2f, R_ARM_LDR_PC_G2, overflow_stack_ptr
> +	add	ip, ip, pc
> +0:	add	ip, ip, #-4
> +1:	add	ip, ip, #0
> +2:	ldr	ip, [ip, #4]
> +
> +	str	sp, [ip, #-4]!			@ Preserve original SP value
> +	mov	sp, ip				@ Switch to overflow stack
> +	pop	{ip}				@ Original SP in IP
> +
> +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
> +	mov	ip, ip				@ mov expected by unwinder
> +	push	{fp, ip, lr, pc}		@ GCC flavor frame record
> +#else
> +	str	ip, [sp, #-8]!			@ store original SP
> +	push	{fpreg, lr}			@ Clang flavor frame record
> +#endif
> +UNWIND( ldr	ip, [r0, #4]	)		@ load exception LR
> +UNWIND( str	ip, [sp, #12]	)		@ store in the frame record
> +	ldr	ip, [r0, #12]			@ reload IP
> +
> +	@ Store the original GPRs to the new stack.
> +	svc_entry uaccess=0, overflow_check=0
> +
> +UNWIND( .save   {sp, pc}	)
> +UNWIND( .save   {fpreg, lr}	)
> +UNWIND( .setfp  fpreg, sp	)
> +
> +	ldr	fpreg, [sp, #S_SP]		@ Add our frame record
> +						@ to the linked list
> +#if defined(CONFIG_UNWINDER_FRAME_POINTER) && defined(CONFIG_CC_IS_GCC)
> +	ldr	r1, [fp, #4]			@ reload SP at entry
> +	add	fp, fp, #12
> +#else
> +	ldr	r1, [fpreg, #8]
> +#endif
> +	str	r1, [sp, #S_SP]			@ store in pt_regs
> +
> +	@ Stash the regs for handle_bad_stack
> +	mov	r0, sp
> +
> +	@ Time to die
> +	bl	handle_bad_stack
> +	nop
> +UNWIND( .fnend			)
> +ENDPROC(__bad_stack)
> +#endif
> +
>   	__INIT
>   
>   /*
> diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
> index ae24dd54e9ef..81df2a3561ca 100644
> --- a/arch/arm/kernel/entry-header.S
> +++ b/arch/arm/kernel/entry-header.S
> @@ -423,3 +423,40 @@ scno	.req	r7		@ syscall number
>   tbl	.req	r8		@ syscall table pointer
>   why	.req	r8		@ Linux syscall (!= 0)
>   tsk	.req	r9		@ current thread_info
> +
> +	.macro	do_overflow_check, frame_size:req
> +#ifdef CONFIG_VMAP_STACK
> +	@
> +	@ Test whether the SP has overflowed. Task and IRQ stacks are aligned
> +	@ so that SP & BIT(THREAD_SIZE_ORDER + PAGE_SHIFT) should always be
> +	@ zero.
> +	@
> +ARM(	tst	sp, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT)	)
> +THUMB(	tst	r1, #1 << (THREAD_SIZE_ORDER + PAGE_SHIFT)	)
> +THUMB(	it	ne						)
> +	bne	.Lstack_overflow_check\@
> +
> +	.pushsection	.text
> +.Lstack_overflow_check\@:
> +	@
> +	@ The stack pointer is not pointing to a valid vmap'ed stack, but it
> +	@ may be pointing into the linear map instead, which may happen if we
> +	@ are already running from the overflow stack. We cannot detect overflow
> +	@ in such cases so just carry on.
> +	@
> +	str	ip, [r0, #12]			@ Stash IP on the mode stack
> +	ldr_l	ip, high_memory			@ Start of VMALLOC space
> +ARM(	cmp	sp, ip			)	@ SP in vmalloc space?
> +THUMB(	cmp	r1, ip			)
> +THUMB(	itt	lo			)
> +	ldrlo	ip, [r0, #12]			@ Restore IP
> +	blo	.Lout\@				@ Carry on
> +
> +THUMB(	sub	r1, sp, r1		)	@ Restore original R1
> +THUMB(	sub	sp, r1			)	@ Restore original SP
> +	add	sp, sp, #\frame_size		@ Undo svc_entry's SP change
> +	b	__bad_stack			@ Handle VMAP stack overflow
> +	.popsection
> +.Lout\@:
> +#endif
> +	.endm
> diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c
> index e05219bca218..5deb40f39999 100644
> --- a/arch/arm/kernel/irq.c
> +++ b/arch/arm/kernel/irq.c
> @@ -56,7 +56,14 @@ static void __init init_irq_stacks(void)
>   	int cpu;
>   
>   	for_each_possible_cpu(cpu) {
> -		stack = (u8 *)__get_free_pages(GFP_KERNEL, THREAD_SIZE_ORDER);
> +		if (!IS_ENABLED(CONFIG_VMAP_STACK))
> +			stack = (u8 *)__get_free_pages(GFP_KERNEL,
> +						       THREAD_SIZE_ORDER);
> +		else
> +			stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
> +					       THREADINFO_GFP, NUMA_NO_NODE,
> +					       __builtin_return_address(0));
> +
>   		if (WARN_ON(!stack))
>   			break;
>   		per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE];
> diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
> index 284a80c0b6e1..039feb7cd590 100644
> --- a/arch/arm/kernel/setup.c
> +++ b/arch/arm/kernel/setup.c
> @@ -141,10 +141,10 @@ EXPORT_SYMBOL(outer_cache);
>   int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN;
>   
>   struct stack {
> -	u32 irq[3];
> -	u32 abt[3];
> -	u32 und[3];
> -	u32 fiq[3];
> +	u32 irq[4];
> +	u32 abt[4];
> +	u32 und[4];
> +	u32 fiq[4];
>   } ____cacheline_aligned;
>   
>   #ifndef CONFIG_CPU_V7M
> diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S
> index 43077e11dafd..803b51e5cba0 100644
> --- a/arch/arm/kernel/sleep.S
> +++ b/arch/arm/kernel/sleep.S
> @@ -67,6 +67,14 @@ ENTRY(__cpu_suspend)
>   	ldr	r4, =cpu_suspend_size
>   #endif
>   	mov	r5, sp			@ current virtual SP
> +#ifdef CONFIG_VMAP_STACK
> +	@ Run the suspend code from the overflow stack so we don't have to rely
> +	@ on vmalloc-to-phys conversions anywhere in the arch suspend code.
> +	@ The original SP value captured in R5 will be restored on the way out.
> +	mov_l	r6, overflow_stack_ptr	@ Base pointer
> +	mrc	p15, 0, r7, c13, c0, 4	@ Get per-CPU offset
> +	ldr	sp, [r6, r7]		@ Address of this CPU's overflow stack
> +#endif
>   	add	r4, r4, #12		@ Space for pgd, virt sp, phys resume fn
>   	sub	sp, sp, r4		@ allocate CPU state on stack
>   	ldr	r3, =sleep_save_sp
> diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
> index b42c446cec9a..b28a705c49cb 100644
> --- a/arch/arm/kernel/traps.c
> +++ b/arch/arm/kernel/traps.c
> @@ -121,7 +121,8 @@ void dump_backtrace_stm(u32 *stack, u32 instruction, const char *loglvl)
>   static int verify_stack(unsigned long sp)
>   {
>   	if (sp < PAGE_OFFSET ||
> -	    (sp > (unsigned long)high_memory && high_memory != NULL))
> +	    (!IS_ENABLED(CONFIG_VMAP_STACK) &&
> +	     sp > (unsigned long)high_memory && high_memory != NULL))
>   		return -EFAULT;
>   
>   	return 0;
> @@ -291,7 +292,8 @@ static int __die(const char *str, int err, struct pt_regs *regs)
>   
>   	if (!user_mode(regs) || in_interrupt()) {
>   		dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp,
> -			 ALIGN(regs->ARM_sp, THREAD_SIZE));
> +			 ALIGN(regs->ARM_sp - THREAD_SIZE, THREAD_ALIGN)
> +			 + THREAD_SIZE);
>   		dump_backtrace(regs, tsk, KERN_EMERG);
>   		dump_instr(KERN_EMERG, regs);
>   	}
> @@ -838,3 +840,77 @@ void __init early_trap_init(void *vectors_base)
>   	 */
>   #endif
>   }
> +
> +#ifdef CONFIG_VMAP_STACK
> +
> +DECLARE_PER_CPU(u8 *, irq_stack_ptr);
> +
> +asmlinkage DEFINE_PER_CPU(u8 *, overflow_stack_ptr);
> +
> +static int __init allocate_overflow_stacks(void)
> +{
> +	u8 *stack;
> +	int cpu;
> +
> +	for_each_possible_cpu(cpu) {
> +		stack = (u8 *)__get_free_page(GFP_KERNEL);
> +		if (WARN_ON(!stack))
> +			return -ENOMEM;
> +		per_cpu(overflow_stack_ptr, cpu) = &stack[OVERFLOW_STACK_SIZE];
> +	}
> +	return 0;
> +}
> +early_initcall(allocate_overflow_stacks);
> +
> +asmlinkage void handle_bad_stack(struct pt_regs *regs)
> +{
> +	unsigned long tsk_stk = (unsigned long)current->stack;
> +	unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
> +	unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr);
> +
> +	console_verbose();
> +	pr_emerg("Insufficient stack space to handle exception!");
> +
> +	pr_emerg("Task stack:     [0x%08lx..0x%08lx]\n",
> +		 tsk_stk, tsk_stk + THREAD_SIZE);
> +	pr_emerg("IRQ stack:      [0x%08lx..0x%08lx]\n",
> +		 irq_stk - THREAD_SIZE, irq_stk);
> +	pr_emerg("Overflow stack: [0x%08lx..0x%08lx]\n",
> +		 ovf_stk - OVERFLOW_STACK_SIZE, ovf_stk);
> +
> +	die("kernel stack overflow", regs, 0);
> +}
> +
> +/*
> + * Normally, we rely on the logic in do_translation_fault() to update stale PMD
> + * entries covering the vmalloc space in a task's page tables when it first
> + * accesses the region in question. Unfortunately, this is not sufficient when
> + * the task stack resides in the vmalloc region, as do_translation_fault() is a
> + * C function that needs a stack to run.
> + *
> + * So we need to ensure that these PMD entries are up to date *before* the MM
> + * switch. As we already have some logic in the MM switch path that takes care
> + * of this, let's trigger it by bumping the counter every time the core vmalloc
> + * code modifies a PMD entry in the vmalloc region.
> + */
> +void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
> +{
> +	if (start > VMALLOC_END || end < VMALLOC_START)
> +		return;
> +
> +	/*
> +	 * This hooks into the core vmalloc code to receive notifications of
> +	 * any PMD level changes that have been made to the kernel page tables.
> +	 * This means it should only be triggered once for every MiB worth of
> +	 * vmalloc space, given that we don't support huge vmalloc/vmap on ARM,
> +	 * and that kernel PMD level table entries are rarely (if ever)
> +	 * updated.
> +	 *
> +	 * This means that the counter is going to max out at ~250 for the
> +	 * typical case. If it overflows, something entirely unexpected has
> +	 * occurred so let's throw a warning if that happens.
> +	 */
> +	WARN_ON(++init_mm.context.vmalloc_seq == UINT_MAX);
> +}
> +
> +#endif
> diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c
> index e8d729975f12..c5ea328c428d 100644
> --- a/arch/arm/kernel/unwind.c
> +++ b/arch/arm/kernel/unwind.c
> @@ -389,7 +389,8 @@ int unwind_frame(struct stackframe *frame)
>   
>   	/* store the highest address on the stack to avoid crossing it*/
>   	ctrl.sp_low = frame->sp;
> -	ctrl.sp_high = ALIGN(ctrl.sp_low, THREAD_SIZE);
> +	ctrl.sp_high = ALIGN(ctrl.sp_low - THREAD_SIZE, THREAD_ALIGN)
> +		       + THREAD_SIZE;
>   
>   	pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__,
>   		 frame->pc, frame->lr, frame->sp);
> diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
> index f02d617e3359..aa12b65a7fd6 100644
> --- a/arch/arm/kernel/vmlinux.lds.S
> +++ b/arch/arm/kernel/vmlinux.lds.S
> @@ -138,12 +138,12 @@ SECTIONS
>   #ifdef CONFIG_STRICT_KERNEL_RWX
>   	. = ALIGN(1<<SECTION_SHIFT);
>   #else
> -	. = ALIGN(THREAD_SIZE);
> +	. = ALIGN(THREAD_ALIGN);
>   #endif
>   	__init_end = .;
>   
>   	_sdata = .;
> -	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)
> +	RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
>   	_edata = .;
>   
>   	BSS_SECTION(0, 0, 0)

Best regards
-- 
Marek Szyprowski, PhD
Samsung R&D Institute Poland




[Index of Archives]     [Linux SoC Development]     [Linux Rockchip Development]     [Linux for Synopsys ARC Processors]    
  • [Linux on Unisoc (RDA Micro) SoCs]     [Linux Actions SoC]     [Linux USB Development]     [Video for Linux]     [Linux Audio Users]     [Linux SCSI]     [Yosemite News]

  •   Powered by Linux