The following commit has been merged into the x86/core branch of tip: Commit-ID: cb6550fd7d3e4e71cb9e6978cc9cea91dcaebd08 Gitweb: https://git.kernel.org/tip/cb6550fd7d3e4e71cb9e6978cc9cea91dcaebd08 Author: Brian Gerst <brgerst@xxxxxxxxx> AuthorDate: Mon, 03 Mar 2025 11:52:37 -05:00 Committer: Ingo Molnar <mingo@xxxxxxxxxx> CommitterDate: Tue, 04 Mar 2025 20:18:02 +01:00 x86/percpu: Move pcpu_hot to percpu hot section Also change the alignment of the percpu hot section: - PERCPU_SECTION(INTERNODE_CACHE_BYTES) + PERCPU_SECTION(L1_CACHE_BYTES) As vSMP will muck with INTERNODE_CACHE_BYTES that invalidates the too-large-section assert we do: ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large") [ mingo: Added INTERNODE_CACHE_BYTES fix & explanation. ] Signed-off-by: Brian Gerst <brgerst@xxxxxxxxx> Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx> Acked-by: Uros Bizjak <ubizjak@xxxxxxxxx> Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx> Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx> Link: https://lore.kernel.org/r/20250303165246.2175811-3-brgerst@xxxxxxxxx --- arch/x86/include/asm/current.h | 28 +++++++++++----------------- arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/vmlinux.lds.S | 5 ++++- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h index bf59538..60bc66e 100644 --- a/arch/x86/include/asm/current.h +++ b/arch/x86/include/asm/current.h @@ -13,32 +13,26 @@ struct task_struct; struct pcpu_hot { - union { - struct { - struct task_struct *current_task; - int preempt_count; - int cpu_number; + struct task_struct *current_task; + int preempt_count; + int cpu_number; #ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING - u64 call_depth; + u64 call_depth; #endif - unsigned long top_of_stack; - void *hardirq_stack_ptr; - u16 softirq_pending; + unsigned long top_of_stack; + void *hardirq_stack_ptr; + u16 softirq_pending; #ifdef CONFIG_X86_64 - bool hardirq_stack_inuse; + bool hardirq_stack_inuse; #else - void *softirq_stack_ptr; + void *softirq_stack_ptr; #endif - }; - u8 pad[64]; - }; }; -static_assert(sizeof(struct pcpu_hot) == 64); -DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot); +DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot); /* const-qualified alias to pcpu_hot, aliased by linker. */ -DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override, +DECLARE_PER_CPU_CACHE_HOT(const struct pcpu_hot __percpu_seg_override, const_pcpu_hot); static __always_inline struct task_struct *get_current(void) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 88a6707..f00870b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -2064,7 +2064,7 @@ static __init int setup_setcpuid(char *arg) } __setup("setcpuid=", setup_setcpuid); -DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = { +DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = { .current_task = &init_task, .preempt_count = INIT_PREEMPT_COUNT, .top_of_stack = TOP_OF_INIT_STACK, diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 1769a71..0ef9870 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -187,6 +187,8 @@ SECTIONS PAGE_ALIGNED_DATA(PAGE_SIZE) + CACHE_HOT_DATA(L1_CACHE_BYTES) + CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES) DATA_DATA @@ -327,7 +329,8 @@ SECTIONS EXIT_DATA } - PERCPU_SECTION(INTERNODE_CACHE_BYTES) + PERCPU_SECTION(L1_CACHE_BYTES) + ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large") RUNTIME_CONST_VARIABLES RUNTIME_CONST(ptr, USER_PTR_MAX)
![]() |