The following commit has been merged into the x86/core branch of tip:
Commit-ID: cb6550fd7d3e4e71cb9e6978cc9cea91dcaebd08
Gitweb: https://git.kernel.org/tip/cb6550fd7d3e4e71cb9e6978cc9cea91dcaebd08
Author: Brian Gerst <brgerst@xxxxxxxxx>
AuthorDate: Mon, 03 Mar 2025 11:52:37 -05:00
Committer: Ingo Molnar <mingo@xxxxxxxxxx>
CommitterDate: Tue, 04 Mar 2025 20:18:02 +01:00
x86/percpu: Move pcpu_hot to percpu hot section
Also change the alignment of the percpu hot section:
- PERCPU_SECTION(INTERNODE_CACHE_BYTES)
+ PERCPU_SECTION(L1_CACHE_BYTES)
As vSMP will muck with INTERNODE_CACHE_BYTES that invalidates the
too-large-section assert we do:
ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large")
[ mingo: Added INTERNODE_CACHE_BYTES fix & explanation. ]
Signed-off-by: Brian Gerst <brgerst@xxxxxxxxx>
Signed-off-by: Ingo Molnar <mingo@xxxxxxxxxx>
Acked-by: Uros Bizjak <ubizjak@xxxxxxxxx>
Cc: Linus Torvalds <torvalds@xxxxxxxxxxxxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Link: https://lore.kernel.org/r/20250303165246.2175811-3-brgerst@xxxxxxxxx
---
arch/x86/include/asm/current.h | 28 +++++++++++-----------------
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/vmlinux.lds.S | 5 ++++-
3 files changed, 16 insertions(+), 19 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index bf59538..60bc66e 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -13,32 +13,26 @@
struct task_struct;
struct pcpu_hot {
- union {
- struct {
- struct task_struct *current_task;
- int preempt_count;
- int cpu_number;
+ struct task_struct *current_task;
+ int preempt_count;
+ int cpu_number;
#ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING
- u64 call_depth;
+ u64 call_depth;
#endif
- unsigned long top_of_stack;
- void *hardirq_stack_ptr;
- u16 softirq_pending;
+ unsigned long top_of_stack;
+ void *hardirq_stack_ptr;
+ u16 softirq_pending;
#ifdef CONFIG_X86_64
- bool hardirq_stack_inuse;
+ bool hardirq_stack_inuse;
#else
- void *softirq_stack_ptr;
+ void *softirq_stack_ptr;
#endif
- };
- u8 pad[64];
- };
};
-static_assert(sizeof(struct pcpu_hot) == 64);
-DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
+DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);
/* const-qualified alias to pcpu_hot, aliased by linker. */
-DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override,
+DECLARE_PER_CPU_CACHE_HOT(const struct pcpu_hot __percpu_seg_override,
const_pcpu_hot);
static __always_inline struct task_struct *get_current(void)
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 88a6707..f00870b 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2064,7 +2064,7 @@ static __init int setup_setcpuid(char *arg)
}
__setup("setcpuid=", setup_setcpuid);
-DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
+DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = {
.current_task = &init_task,
.preempt_count = INIT_PREEMPT_COUNT,
.top_of_stack = TOP_OF_INIT_STACK,
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 1769a71..0ef9870 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -187,6 +187,8 @@ SECTIONS
PAGE_ALIGNED_DATA(PAGE_SIZE)
+ CACHE_HOT_DATA(L1_CACHE_BYTES)
+
CACHELINE_ALIGNED_DATA(L1_CACHE_BYTES)
DATA_DATA
@@ -327,7 +329,8 @@ SECTIONS
EXIT_DATA
}
- PERCPU_SECTION(INTERNODE_CACHE_BYTES)
+ PERCPU_SECTION(L1_CACHE_BYTES)
+ ASSERT(__per_cpu_hot_end - __per_cpu_hot_start <= 64, "percpu cache hot section too large")
RUNTIME_CONST_VARIABLES
RUNTIME_CONST(ptr, USER_PTR_MAX)
|