在2022年6月1日六月 上午11:00，Huacai Chen写道： > LoongArch-based procesors have 4, 8 or 16 cores per package. This patch > adds multi-processor (SMP) support for LoongArch. > > Reviewed-by: WANG Xuerui <git@xxxxxxxxxx> > Signed-off-by: Huacai Chen <chenhuacai@xxxxxxxxxxx> Reviewed-by: Jiaxun Yang <jiaxun.yang@xxxxxxxxxxx> CSR IPI makes things much more easily than MIPS era. Thanks. > --- > arch/loongarch/Kconfig | 40 +- > arch/loongarch/include/asm/atomic.h | 4 + > arch/loongarch/include/asm/barrier.h | 108 ++++ > arch/loongarch/include/asm/cmpxchg.h | 1 + > arch/loongarch/include/asm/futex.h | 1 + > arch/loongarch/include/asm/hardirq.h | 2 + > arch/loongarch/include/asm/irq.h | 2 + > arch/loongarch/include/asm/percpu.h | 194 +++++++ > arch/loongarch/include/asm/pgtable.h | 21 + > arch/loongarch/include/asm/smp.h | 124 ++++ > arch/loongarch/include/asm/stackframe.h | 17 +- > arch/loongarch/include/asm/tlbflush.h | 13 + > arch/loongarch/include/asm/topology.h | 7 +- > arch/loongarch/kernel/Makefile | 2 + > arch/loongarch/kernel/acpi.c | 69 +++ > arch/loongarch/kernel/asm-offsets.c | 10 + > arch/loongarch/kernel/head.S | 30 + > arch/loongarch/kernel/irq.c | 13 +- > arch/loongarch/kernel/proc.c | 5 + > arch/loongarch/kernel/process.c | 7 + > arch/loongarch/kernel/reset.c | 12 + > arch/loongarch/kernel/setup.c | 26 + > arch/loongarch/kernel/smp.c | 735 ++++++++++++++++++++++++ > arch/loongarch/kernel/topology.c | 43 +- > arch/loongarch/kernel/vmlinux.lds.S | 4 + > arch/loongarch/mm/tlbex.S | 69 +++ > include/linux/cpuhotplug.h | 1 + > 27 files changed, 1550 insertions(+), 10 deletions(-) > create mode 100644 arch/loongarch/include/asm/smp.h > create mode 100644 arch/loongarch/kernel/smp.c > > diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig > index d6ac80cf3922..b252a51946e3 100644 > --- a/arch/loongarch/Kconfig > +++ b/arch/loongarch/Kconfig > @@ -64,6 +64,7 @@ config LOONGARCH > select GENERIC_LIB_UCMPDI2 > select GENERIC_PCI_IOMAP > select GENERIC_SCHED_CLOCK > + select GENERIC_SMP_IDLE_THREAD > select GENERIC_TIME_VSYSCALL > select GPIOLIB > select HAVE_ARCH_AUDITSYSCALL > @@ -92,7 +93,7 @@ config LOONGARCH > select HAVE_RSEQ > select HAVE_SYSCALL_TRACEPOINTS > select HAVE_TIF_NOHZ > - select HAVE_VIRT_CPU_ACCOUNTING_GEN > + select HAVE_VIRT_CPU_ACCOUNTING_GEN if !SMP > select IRQ_FORCED_THREADING > select IRQ_LOONGARCH_CPU > select MODULES_USE_ELF_RELA if MODULES > @@ -306,6 +307,43 @@ config EFI_STUB > This kernel feature allows the kernel to be loaded directly by > EFI firmware without the use of a bootloader. > > +config SMP > + bool "Multi-Processing support" > + help > + This enables support for systems with more than one CPU. If you have > + a system with only one CPU, say N. If you have a system with more > + than one CPU, say Y. > + > + If you say N here, the kernel will run on uni- and multiprocessor > + machines, but will use only one CPU of a multiprocessor machine. If > + you say Y here, the kernel will run on many, but not all, > + uniprocessor machines. On a uniprocessor machine, the kernel > + will run faster if you say N here. > + > + See also the SMP-HOWTO available at > <http://www.tldp.org/docs.html#howto>. > + > + If you don't know what to do here, say N. > + > +config HOTPLUG_CPU > + bool "Support for hot-pluggable CPUs" > + depends on SMP > + select GENERIC_IRQ_MIGRATION > + help > + Say Y here to allow turning CPUs off and on. CPUs can be > + controlled through /sys/devices/system/cpu. > + (Note: power management support will enable this option > + automatically on SMP systems. ) > + Say N if you want to disable CPU hotplug. > + > +config NR_CPUS > + int "Maximum number of CPUs (2-256)" > + range 2 256 > + depends on SMP > + default "64" > + help > + This allows you to specify the maximum number of CPUs which this > + kernel will support. > + > config FORCE_MAX_ZONEORDER > int "Maximum zone order" > range 14 64 if PAGE_SIZE_64KB > diff --git a/arch/loongarch/include/asm/atomic.h > b/arch/loongarch/include/asm/atomic.h > index 932352342b12..979367ad4e2c 100644 > --- a/arch/loongarch/include/asm/atomic.h > +++ b/arch/loongarch/include/asm/atomic.h > @@ -162,6 +162,7 @@ static inline int arch_atomic_sub_if_positive(int > i, atomic_t *v) > " sc.w %1, %2 \n" > " beq $zero, %1, 1b \n" > "2: \n" > + __WEAK_LLSC_MB > : "=&r" (result), "=&r" (temp), > "+" GCC_OFF_SMALL_ASM() (v->counter) > : "I" (-i)); > @@ -174,6 +175,7 @@ static inline int arch_atomic_sub_if_positive(int > i, atomic_t *v) > " sc.w %1, %2 \n" > " beq $zero, %1, 1b \n" > "2: \n" > + __WEAK_LLSC_MB > : "=&r" (result), "=&r" (temp), > "+" GCC_OFF_SMALL_ASM() (v->counter) > : "r" (i)); > @@ -323,6 +325,7 @@ static inline long > arch_atomic64_sub_if_positive(long i, atomic64_t *v) > " sc.d %1, %2 \n" > " beq %1, $zero, 1b \n" > "2: \n" > + __WEAK_LLSC_MB > : "=&r" (result), "=&r" (temp), > "+" GCC_OFF_SMALL_ASM() (v->counter) > : "I" (-i)); > @@ -335,6 +338,7 @@ static inline long > arch_atomic64_sub_if_positive(long i, atomic64_t *v) > " sc.d %1, %2 \n" > " beq %1, $zero, 1b \n" > "2: \n" > + __WEAK_LLSC_MB > : "=&r" (result), "=&r" (temp), > "+" GCC_OFF_SMALL_ASM() (v->counter) > : "r" (i)); > diff --git a/arch/loongarch/include/asm/barrier.h > b/arch/loongarch/include/asm/barrier.h > index e57571bcaf4f..b6517eeeb141 100644 > --- a/arch/loongarch/include/asm/barrier.h > +++ b/arch/loongarch/include/asm/barrier.h > @@ -18,6 +18,19 @@ > #define mb() fast_mb() > #define iob() fast_iob() > > +#define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory") > +#define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory") > +#define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory") > + > +#ifdef CONFIG_SMP > +#define __WEAK_LLSC_MB " dbar 0 \n" > +#else > +#define __WEAK_LLSC_MB " \n" > +#endif > + > +#define __smp_mb__before_atomic() barrier() > +#define __smp_mb__after_atomic() barrier() > + > /** > * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 > otherwise > * @index: array element index > @@ -46,6 +59,101 @@ static inline unsigned long > array_index_mask_nospec(unsigned long index, > return mask; > } > > +#define __smp_load_acquire(p) \ > +({ \ > + union { typeof(*p) __val; char __c[1]; } __u; \ > + unsigned long __tmp = 0; \ > + compiletime_assert_atomic_type(*p); \ > + switch (sizeof(*p)) { \ > + case 1: \ > + *(__u8 *)__u.__c = *(volatile __u8 *)p; \ > + __smp_mb(); \ > + break; \ > + case 2: \ > + *(__u16 *)__u.__c = *(volatile __u16 *)p; \ > + __smp_mb(); \ > + break; \ > + case 4: \ > + __asm__ __volatile__( \ > + "amor_db.w %[val], %[tmp], %[mem] \n" \ > + : [val] "=&r" (*(__u32 *)__u.__c) \ > + : [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \ > + : "memory"); \ > + break; \ > + case 8: \ > + __asm__ __volatile__( \ > + "amor_db.d %[val], %[tmp], %[mem] \n" \ > + : [val] "=&r" (*(__u64 *)__u.__c) \ > + : [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \ > + : "memory"); \ > + break; \ > + } \ > + (typeof(*p))__u.__val; \ > +}) > + > +#define __smp_store_release(p, v) \ > +do { \ > + union { typeof(*p) __val; char __c[1]; } __u = \ > + { .__val = (__force typeof(*p)) (v) }; \ > + unsigned long __tmp; \ > + compiletime_assert_atomic_type(*p); \ > + switch (sizeof(*p)) { \ > + case 1: \ > + __smp_mb(); \ > + *(volatile __u8 *)p = *(__u8 *)__u.__c; \ > + break; \ > + case 2: \ > + __smp_mb(); \ > + *(volatile __u16 *)p = *(__u16 *)__u.__c; \ > + break; \ > + case 4: \ > + __asm__ __volatile__( \ > + "amswap_db.w %[tmp], %[val], %[mem] \n" \ > + : [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \ > + : [val] "r" (*(__u32 *)__u.__c) \ > + : ); \ > + break; \ > + case 8: \ > + __asm__ __volatile__( \ > + "amswap_db.d %[tmp], %[val], %[mem] \n" \ > + : [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \ > + : [val] "r" (*(__u64 *)__u.__c) \ > + : ); \ > + break; \ > + } \ > +} while (0) > + > +#define __smp_store_mb(p, v) \ > +do { \ > + union { typeof(p) __val; char __c[1]; } __u = \ > + { .__val = (__force typeof(p)) (v) }; \ > + unsigned long __tmp; \ > + switch (sizeof(p)) { \ > + case 1: \ > + *(volatile __u8 *)&p = *(__u8 *)__u.__c; \ > + __smp_mb(); \ > + break; \ > + case 2: \ > + *(volatile __u16 *)&p = *(__u16 *)__u.__c; \ > + __smp_mb(); \ > + break; \ > + case 4: \ > + __asm__ __volatile__( \ > + "amswap_db.w %[tmp], %[val], %[mem] \n" \ > + : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \ > + : [val] "r" (*(__u32 *)__u.__c) \ > + : ); \ > + break; \ > + case 8: \ > + __asm__ __volatile__( \ > + "amswap_db.d %[tmp], %[val], %[mem] \n" \ > + : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \ > + : [val] "r" (*(__u64 *)__u.__c) \ > + : ); \ > + break; \ > + } \ > +} while (0) > + > #include <asm-generic/barrier.h> > > #endif /* __ASM_BARRIER_H */ > diff --git a/arch/loongarch/include/asm/cmpxchg.h > b/arch/loongarch/include/asm/cmpxchg.h > index 48613b872bc8..75b3a4478652 100644 > --- a/arch/loongarch/include/asm/cmpxchg.h > +++ b/arch/loongarch/include/asm/cmpxchg.h > @@ -59,6 +59,7 @@ static inline unsigned long __xchg(volatile void > *ptr, unsigned long x, > " " st " $t0, %1 \n" \ > " beq $zero, $t0, 1b \n" \ > "2: \n" \ > + __WEAK_LLSC_MB \ > : "=&r" (__ret), "=ZB"(*m) \ > : "ZB"(*m), "Jr" (old), "Jr" (new) \ > : "t0", "memory"); \ > diff --git a/arch/loongarch/include/asm/futex.h > b/arch/loongarch/include/asm/futex.h > index b27d55f92db7..9de8231694ec 100644 > --- a/arch/loongarch/include/asm/futex.h > +++ b/arch/loongarch/include/asm/futex.h > @@ -86,6 +86,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user > *uaddr, u32 oldval, u32 newv > "2: sc.w $t0, %2 \n" > " beq $zero, $t0, 1b \n" > "3: \n" > + __WEAK_LLSC_MB > " .section .fixup,\"ax\" \n" > "4: li.d %0, %6 \n" > " b 3b \n" > diff --git a/arch/loongarch/include/asm/hardirq.h > b/arch/loongarch/include/asm/hardirq.h > index d32f83938880..befe8184aa08 100644 > --- a/arch/loongarch/include/asm/hardirq.h > +++ b/arch/loongarch/include/asm/hardirq.h > @@ -21,4 +21,6 @@ typedef struct { > > DECLARE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat); > > +#define __ARCH_IRQ_STAT > + > #endif /* _ASM_HARDIRQ_H */ > diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h > index f3f1baf7027c..ace3ea6da72e 100644 > --- a/arch/loongarch/include/asm/irq.h > +++ b/arch/loongarch/include/asm/irq.h > @@ -125,6 +125,8 @@ extern struct irq_domain *pch_lpc_domain; > extern struct irq_domain *pch_msi_domain[MAX_IO_PICS]; > extern struct irq_domain *pch_pic_domain[MAX_IO_PICS]; > > +extern irqreturn_t loongson3_ipi_interrupt(int irq, void *dev); > + > #include <asm-generic/irq.h> > > #endif /* _ASM_IRQ_H */ > diff --git a/arch/loongarch/include/asm/percpu.h > b/arch/loongarch/include/asm/percpu.h > index b03d8f8b9fd3..34f15a6fb1e7 100644 > --- a/arch/loongarch/include/asm/percpu.h > +++ b/arch/loongarch/include/asm/percpu.h > @@ -5,6 +5,8 @@ > #ifndef __ASM_PERCPU_H > #define __ASM_PERCPU_H > > +#include <asm/cmpxchg.h> > + > /* Use r21 for fast access */ > register unsigned long __my_cpu_offset __asm__("$r21"); > > @@ -15,6 +17,198 @@ static inline void set_my_cpu_offset(unsigned long off) > } > #define __my_cpu_offset __my_cpu_offset > > +#define PERCPU_OP(op, asm_op, c_op) \ > +static inline unsigned long __percpu_##op(void *ptr, \ > + unsigned long val, int size) \ > +{ \ > + unsigned long ret; \ > + \ > + switch (size) { \ > + case 4: \ > + __asm__ __volatile__( \ > + "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \ > + : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \ > + : [val] "r" (val)); \ > + break; \ > + case 8: \ > + __asm__ __volatile__( \ > + "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \ > + : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \ > + : [val] "r" (val)); \ > + break; \ > + default: \ > + ret = 0; \ > + BUILD_BUG(); \ > + } \ > + \ > + return ret c_op val; \ > +} > + > +PERCPU_OP(add, add, +) > +PERCPU_OP(and, and, &) > +PERCPU_OP(or, or, |) > +#undef PERCPU_OP > + > +static inline unsigned long __percpu_read(void *ptr, int size) > +{ > + unsigned long ret; > + > + switch (size) { > + case 1: > + __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n" > + : [ret] "=&r"(ret) > + : [ptr] "r"(ptr) > + : "memory"); > + break; > + case 2: > + __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n" > + : [ret] "=&r"(ret) > + : [ptr] "r"(ptr) > + : "memory"); > + break; > + case 4: > + __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n" > + : [ret] "=&r"(ret) > + : [ptr] "r"(ptr) > + : "memory"); > + break; > + case 8: > + __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n" > + : [ret] "=&r"(ret) > + : [ptr] "r"(ptr) > + : "memory"); > + break; > + default: > + ret = 0; > + BUILD_BUG(); > + } > + > + return ret; > +} > + > +static inline void __percpu_write(void *ptr, unsigned long val, int size) > +{ > + switch (size) { > + case 1: > + __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n" > + : > + : [val] "r" (val), [ptr] "r" (ptr) > + : "memory"); > + break; > + case 2: > + __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n" > + : > + : [val] "r" (val), [ptr] "r" (ptr) > + : "memory"); > + break; > + case 4: > + __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n" > + : > + : [val] "r" (val), [ptr] "r" (ptr) > + : "memory"); > + break; > + case 8: > + __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n" > + : > + : [val] "r" (val), [ptr] "r" (ptr) > + : "memory"); > + break; > + default: > + BUILD_BUG(); > + } > +} > + > +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, > + int size) > +{ > + switch (size) { > + case 4: > + return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val); > + > + case 8: > + return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val); > + > + default: > + BUILD_BUG(); > + } > + > + return 0; > +} > + > +/* this_cpu_cmpxchg */ > +#define _protect_cmpxchg_local(pcp, o, n) \ > +({ \ > + typeof(*raw_cpu_ptr(&(pcp))) __ret; \ > + preempt_disable_notrace(); \ > + __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ > + preempt_enable_notrace(); \ > + __ret; \ > +}) > + > +#define _percpu_read(pcp) \ > +({ \ > + typeof(pcp) __retval; \ > + __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \ > + __retval; \ > +}) > + > +#define _percpu_write(pcp, val) \ > +do { \ > + __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \ > +} while (0) \ > + > +#define _pcp_protect(operation, pcp, val) \ > +({ \ > + typeof(pcp) __retval; \ > + preempt_disable_notrace(); \ > + __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \ > + (val), sizeof(pcp)); \ > + preempt_enable_notrace(); \ > + __retval; \ > +}) > + > +#define _percpu_add(pcp, val) \ > + _pcp_protect(__percpu_add, pcp, val) > + > +#define _percpu_add_return(pcp, val) _percpu_add(pcp, val) > + > +#define _percpu_and(pcp, val) \ > + _pcp_protect(__percpu_and, pcp, val) > + > +#define _percpu_or(pcp, val) \ > + _pcp_protect(__percpu_or, pcp, val) > + > +#define _percpu_xchg(pcp, val) ((typeof(pcp)) \ > + _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))) > + > +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) > +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) > + > +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) > +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) > + > +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) > +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) > + > +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) > +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) > + > +#define this_cpu_read_1(pcp) _percpu_read(pcp) > +#define this_cpu_read_2(pcp) _percpu_read(pcp) > +#define this_cpu_read_4(pcp) _percpu_read(pcp) > +#define this_cpu_read_8(pcp) _percpu_read(pcp) > + > +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) > +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) > +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) > +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) > + > +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) > +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) > + > +#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) > +#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) > + > #include <asm-generic/percpu.h> > > #endif /* __ASM_PERCPU_H */ > diff --git a/arch/loongarch/include/asm/pgtable.h > b/arch/loongarch/include/asm/pgtable.h > index 8920dd8b297b..5e33987d0a13 100644 > --- a/arch/loongarch/include/asm/pgtable.h > +++ b/arch/loongarch/include/asm/pgtable.h > @@ -279,8 +279,29 @@ static inline void set_pte(pte_t *ptep, pte_t > pteval) > * Make sure the buddy is global too (if it's !none, > * it better already be global) > */ > +#ifdef CONFIG_SMP > + /* > + * For SMP, multiple CPUs can race, so we need to do > + * this atomically. > + */ > + unsigned long page_global = _PAGE_GLOBAL; > + unsigned long tmp; > + > + __asm__ __volatile__ ( > + "1:" __LL "%[tmp], %[buddy] \n" > + " bnez %[tmp], 2f \n" > + " or %[tmp], %[tmp], %[global] \n" > + __SC "%[tmp], %[buddy] \n" > + " beqz %[tmp], 1b \n" > + " nop \n" > + "2: \n" > + __WEAK_LLSC_MB > + : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp) > + : [global] "r" (page_global)); > +#else /* !CONFIG_SMP */ > if (pte_none(*buddy)) > pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL; > +#endif /* CONFIG_SMP */ > } > } > > diff --git a/arch/loongarch/include/asm/smp.h > b/arch/loongarch/include/asm/smp.h > new file mode 100644 > index 000000000000..551e1f37c705 > --- /dev/null > +++ b/arch/loongarch/include/asm/smp.h > @@ -0,0 +1,124 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +/* > + * Author: Huacai Chen <chenhuacai@xxxxxxxxxxx> > + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited > + */ > +#ifndef __ASM_SMP_H > +#define __ASM_SMP_H > + > +#include <linux/atomic.h> > +#include <linux/bitops.h> > +#include <linux/linkage.h> > +#include <linux/smp.h> > +#include <linux/threads.h> > +#include <linux/cpumask.h> > + > +void loongson3_smp_setup(void); > +void loongson3_prepare_cpus(unsigned int max_cpus); > +void loongson3_boot_secondary(int cpu, struct task_struct *idle); > +void loongson3_init_secondary(void); > +void loongson3_smp_finish(void); > +void loongson3_send_ipi_single(int cpu, unsigned int action); > +void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int > action); > +#ifdef CONFIG_HOTPLUG_CPU > +int loongson3_cpu_disable(void); > +void loongson3_cpu_die(unsigned int cpu); > +#endif > + > +#ifdef CONFIG_SMP > + > +static inline void plat_smp_setup(void) > +{ > + loongson3_smp_setup(); > +} > + > +#else /* !CONFIG_SMP */ > + > +static inline void plat_smp_setup(void) { } > + > +#endif /* !CONFIG_SMP */ > + > +extern int smp_num_siblings; > +extern int num_processors; > +extern int disabled_cpus; > +extern cpumask_t cpu_sibling_map[]; > +extern cpumask_t cpu_core_map[]; > +extern cpumask_t cpu_foreign_map[]; > + > +static inline int raw_smp_processor_id(void) > +{ > +#if defined(__VDSO__) > + extern int vdso_smp_processor_id(void) > + __compiletime_error("VDSO should not call smp_processor_id()"); > + return vdso_smp_processor_id(); > +#else > + return current_thread_info()->cpu; > +#endif > +} > +#define raw_smp_processor_id raw_smp_processor_id > + > +/* Map from cpu id to sequential logical cpu number. This will only > + * not be idempotent when cpus failed to come on-line. */ > +extern int __cpu_number_map[NR_CPUS]; > +#define cpu_number_map(cpu) __cpu_number_map[cpu] > + > +/* The reverse map from sequential logical cpu number to cpu id. */ > +extern int __cpu_logical_map[NR_CPUS]; > +#define cpu_logical_map(cpu) __cpu_logical_map[cpu] > + > +#define cpu_physical_id(cpu) cpu_logical_map(cpu) > + > +#define SMP_BOOT_CPU 0x1 > +#define SMP_RESCHEDULE 0x2 > +#define SMP_CALL_FUNCTION 0x4 > + > +struct secondary_data { > + unsigned long stack; > + unsigned long thread_info; > +}; > +extern struct secondary_data cpuboot_data; > + > +extern asmlinkage void smpboot_entry(void); > + > +extern void calculate_cpu_foreign_map(void); > + > +/* > + * Generate IPI list text > + */ > +extern void show_ipi_list(struct seq_file *p, int prec); > + > +/* > + * This function sends a 'reschedule' IPI to another CPU. > + * it goes straight through and wastes no time serializing > + * anything. Worst case is that we lose a reschedule ... > + */ > +static inline void smp_send_reschedule(int cpu) > +{ > + loongson3_send_ipi_single(cpu, SMP_RESCHEDULE); > +} > + > +static inline void arch_send_call_function_single_ipi(int cpu) > +{ > + loongson3_send_ipi_single(cpu, SMP_CALL_FUNCTION); > +} > + > +static inline void arch_send_call_function_ipi_mask(const struct > cpumask *mask) > +{ > + loongson3_send_ipi_mask(mask, SMP_CALL_FUNCTION); > +} > + > +#ifdef CONFIG_HOTPLUG_CPU > +static inline int __cpu_disable(void) > +{ > + return loongson3_cpu_disable(); > +} > + > +static inline void __cpu_die(unsigned int cpu) > +{ > + loongson3_cpu_die(cpu); > +} > + > +extern void play_dead(void); > +#endif > + > +#endif /* __ASM_SMP_H */ > diff --git a/arch/loongarch/include/asm/stackframe.h > b/arch/loongarch/include/asm/stackframe.h > index 44151b878d00..4ca953062b5b 100644 > --- a/arch/loongarch/include/asm/stackframe.h > +++ b/arch/loongarch/include/asm/stackframe.h > @@ -77,17 +77,24 @@ > * new value in sp. > */ > .macro get_saved_sp docfi=0 > - la.abs t1, kernelsp > - move t0, sp > + la.abs t1, kernelsp > +#ifdef CONFIG_SMP > + csrrd t0, PERCPU_BASE_KS > + LONG_ADD t1, t1, t0 > +#endif > + move t0, sp > .if \docfi > .cfi_register sp, t0 > .endif > - LONG_L sp, t1, 0 > + LONG_L sp, t1, 0 > .endm > > .macro set_saved_sp stackp temp temp2 > - la.abs \temp, kernelsp > - LONG_S \stackp, \temp, 0 > + la.abs \temp, kernelsp > +#ifdef CONFIG_SMP > + LONG_ADD \temp, \temp, u0 > +#endif > + LONG_S \stackp, \temp, 0 > .endm > > .macro SAVE_SOME docfi=0 > diff --git a/arch/loongarch/include/asm/tlbflush.h > b/arch/loongarch/include/asm/tlbflush.h > index 36bd6d11dc2d..a0785e590681 100644 > --- a/arch/loongarch/include/asm/tlbflush.h > +++ b/arch/loongarch/include/asm/tlbflush.h > @@ -25,6 +25,17 @@ extern void local_flush_tlb_kernel_range(unsigned > long start, unsigned long end) > extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned > long page); > extern void local_flush_tlb_one(unsigned long vaddr); > > +#ifdef CONFIG_SMP > + > +extern void flush_tlb_all(void); > +extern void flush_tlb_mm(struct mm_struct *); > +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long, > unsigned long); > +extern void flush_tlb_kernel_range(unsigned long, unsigned long); > +extern void flush_tlb_page(struct vm_area_struct *, unsigned long); > +extern void flush_tlb_one(unsigned long vaddr); > + > +#else /* CONFIG_SMP */ > + > #define flush_tlb_all() local_flush_tlb_all() > #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) > #define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, > vmaddr, end) > @@ -32,4 +43,6 @@ extern void local_flush_tlb_one(unsigned long vaddr); > #define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page) > #define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr) > > +#endif /* CONFIG_SMP */ > + > #endif /* __ASM_TLBFLUSH_H */ > diff --git a/arch/loongarch/include/asm/topology.h > b/arch/loongarch/include/asm/topology.h > index 9ac71a25207a..da135841e5b1 100644 > --- a/arch/loongarch/include/asm/topology.h > +++ b/arch/loongarch/include/asm/topology.h > @@ -7,7 +7,12 @@ > > #include <linux/smp.h> > > -#define cpu_logical_map(cpu) 0 > +#ifdef CONFIG_SMP > +#define topology_physical_package_id(cpu) (cpu_data[cpu].package) > +#define topology_core_id(cpu) (cpu_data[cpu].core) > +#define topology_core_cpumask(cpu) (&cpu_core_map[cpu]) > +#define topology_sibling_cpumask(cpu) (&cpu_sibling_map[cpu]) > +#endif > > #include <asm-generic/topology.h> > > diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile > index e5a3b2fb9961..2cb6f698716a 100644 > --- a/arch/loongarch/kernel/Makefile > +++ b/arch/loongarch/kernel/Makefile > @@ -18,4 +18,6 @@ obj-$(CONFIG_MODULES) += module.o module-sections.o > > obj-$(CONFIG_PROC_FS) += proc.o > > +obj-$(CONFIG_SMP) += smp.o > + > CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) > diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c > index a644220bb426..181c59493b63 100644 > --- a/arch/loongarch/kernel/acpi.c > +++ b/arch/loongarch/kernel/acpi.c > @@ -137,8 +137,44 @@ void __init acpi_boot_table_init(void) > } > } > > +static int set_processor_mask(u32 id, u32 flags) > +{ > + > + int cpu, cpuid = id; > + > + if (num_processors >= nr_cpu_ids) { > + pr_warn(PREFIX "nr_cpus/possible_cpus limit of %i reached." > + " processor 0x%x ignored.\n", nr_cpu_ids, cpuid); > + > + return -ENODEV; > + > + } > + if (cpuid == loongson_sysconf.boot_cpu_id) > + cpu = 0; > + else > + cpu = cpumask_next_zero(-1, cpu_present_mask); > + > + if (flags & ACPI_MADT_ENABLED) { > + num_processors++; > + set_cpu_possible(cpu, true); > + set_cpu_present(cpu, true); > + __cpu_number_map[cpuid] = cpu; > + __cpu_logical_map[cpu] = cpuid; > + } else > + disabled_cpus++; > + > + return cpu; > +} > + > static void __init acpi_process_madt(void) > { > + int i; > + > + for (i = 0; i < NR_CPUS; i++) { > + __cpu_number_map[i] = -1; > + __cpu_logical_map[i] = -1; > + } > + > loongson_sysconf.nr_cpus = num_processors; > } > > @@ -167,3 +203,36 @@ void __init > arch_reserve_mem_area(acpi_physical_address addr, size_t size) > { > memblock_reserve(addr, size); > } > + > +#ifdef CONFIG_ACPI_HOTPLUG_CPU > + > +#include <acpi/processor.h> > + > +int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, > int *pcpu) > +{ > + int cpu; > + > + cpu = set_processor_mask(physid, ACPI_MADT_ENABLED); > + if (cpu < 0) { > + pr_info(PREFIX "Unable to map lapic to logical cpu number\n"); > + return cpu; > + } > + > + *pcpu = cpu; > + > + return 0; > +} > +EXPORT_SYMBOL(acpi_map_cpu); > + > +int acpi_unmap_cpu(int cpu) > +{ > + set_cpu_present(cpu, false); > + num_processors--; > + > + pr_info("cpu%d hot remove!\n", cpu); > + > + return 0; > +} > +EXPORT_SYMBOL(acpi_unmap_cpu); > + > +#endif /* CONFIG_ACPI_HOTPLUG_CPU */ > diff --git a/arch/loongarch/kernel/asm-offsets.c > b/arch/loongarch/kernel/asm-offsets.c > index 3531e3c60a6e..bfb65eb2844f 100644 > --- a/arch/loongarch/kernel/asm-offsets.c > +++ b/arch/loongarch/kernel/asm-offsets.c > @@ -252,3 +252,13 @@ void output_signal_defines(void) > DEFINE(_SIGXFSZ, SIGXFSZ); > BLANK(); > } > + > +#ifdef CONFIG_SMP > +void output_smpboot_defines(void) > +{ > + COMMENT("Linux smp cpu boot offsets."); > + OFFSET(CPU_BOOT_STACK, secondary_data, stack); > + OFFSET(CPU_BOOT_TINFO, secondary_data, thread_info); > + BLANK(); > +} > +#endif > diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S > index 4cb79e9c70f5..8ca45cf17433 100644 > --- a/arch/loongarch/kernel/head.S > +++ b/arch/loongarch/kernel/head.S > @@ -96,4 +96,34 @@ SYM_CODE_START(kernel_entry) # kernel entry point > > SYM_CODE_END(kernel_entry) > > +#ifdef CONFIG_SMP > + > +/* > + * SMP slave cpus entry point. Board specific code for bootstrap calls this > + * function after setting up the stack and tp registers. > + */ > +SYM_CODE_START(smpboot_entry) > + li.d t0, CSR_DMW0_INIT # UC, PLV0 > + csrwr t0, LOONGARCH_CSR_DMWIN0 > + li.d t0, CSR_DMW1_INIT # CA, PLV0 > + csrwr t0, LOONGARCH_CSR_DMWIN1 > + li.w t0, 0xb0 # PLV=0, IE=0, PG=1 > + csrwr t0, LOONGARCH_CSR_CRMD > + li.w t0, 0x04 # PLV=0, PIE=1, PWE=0 > + csrwr t0, LOONGARCH_CSR_PRMD > + li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 > + csrwr t0, LOONGARCH_CSR_EUEN > + > + la.abs t0, cpuboot_data > + ld.d sp, t0, CPU_BOOT_STACK > + ld.d tp, t0, CPU_BOOT_TINFO > + > + la.abs t0, 0f > + jirl zero, t0, 0 > +0: > + bl start_secondary > +SYM_CODE_END(smpboot_entry) > + > +#endif /* CONFIG_SMP */ > + > SYM_ENTRY(kernel_entry_end, SYM_L_GLOBAL, SYM_A_NONE) > diff --git a/arch/loongarch/kernel/irq.c b/arch/loongarch/kernel/irq.c > index 9bd07edf0bce..4b671d305ede 100644 > --- a/arch/loongarch/kernel/irq.c > +++ b/arch/loongarch/kernel/irq.c > @@ -47,13 +47,17 @@ asmlinkage void spurious_interrupt(void) > > int arch_show_interrupts(struct seq_file *p, int prec) > { > +#ifdef CONFIG_SMP > + show_ipi_list(p, prec); > +#endif > seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); > return 0; > } > > void __init init_IRQ(void) > { > - int i; > + int i, r, ipi_irq; > + static int ipi_dummy_dev; > unsigned int order = get_order(IRQ_STACK_SIZE); > struct page *page; > > @@ -61,6 +65,13 @@ void __init init_IRQ(void) > clear_csr_estat(ESTATF_IP); > > irqchip_init(); > +#ifdef CONFIG_SMP > + ipi_irq = EXCCODE_IPI - EXCCODE_INT_START; > + irq_set_percpu_devid(ipi_irq); > + r = request_percpu_irq(ipi_irq, loongson3_ipi_interrupt, "IPI", > &ipi_dummy_dev); > + if (r < 0) > + panic("IPI IRQ request failed\n"); > +#endif > > for (i = 0; i < NR_IRQS; i++) > irq_set_noprobe(i); > diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c > index d25592a29196..1effc73850fe 100644 > --- a/arch/loongarch/kernel/proc.c > +++ b/arch/loongarch/kernel/proc.c > @@ -35,6 +35,11 @@ static int show_cpuinfo(struct seq_file *m, void *v) > unsigned int fp_version = cpu_data[n].fpu_vers; > struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; > > +#ifdef CONFIG_SMP > + if (!cpu_online(n)) > + return 0; > +#endif > + > /* > * For the first processor also print the system type > */ > diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c > index 54020223068f..6d944d65f600 100644 > --- a/arch/loongarch/kernel/process.c > +++ b/arch/loongarch/kernel/process.c > @@ -53,6 +53,13 @@ > unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; > EXPORT_SYMBOL(boot_option_idle_override); > > +#ifdef CONFIG_HOTPLUG_CPU > +void arch_cpu_idle_dead(void) > +{ > + play_dead(); > +} > +#endif > + > asmlinkage void ret_from_fork(void); > asmlinkage void ret_from_kernel_thread(void); > > diff --git a/arch/loongarch/kernel/reset.c b/arch/loongarch/kernel/reset.c > index ef484ce43c5c..2b86469e4718 100644 > --- a/arch/loongarch/kernel/reset.c > +++ b/arch/loongarch/kernel/reset.c > @@ -65,16 +65,28 @@ EXPORT_SYMBOL(pm_power_off); > > void machine_halt(void) > { > +#ifdef CONFIG_SMP > + preempt_disable(); > + smp_send_stop(); > +#endif > default_halt(); > } > > void machine_power_off(void) > { > +#ifdef CONFIG_SMP > + preempt_disable(); > + smp_send_stop(); > +#endif > pm_power_off(); > } > > void machine_restart(char *command) > { > +#ifdef CONFIG_SMP > + preempt_disable(); > + smp_send_stop(); > +#endif > do_kernel_restart(command); > pm_restart(); > } > diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c > index 29f3b82cd0a5..34a3011f679e 100644 > --- a/arch/loongarch/kernel/setup.c > +++ b/arch/loongarch/kernel/setup.c > @@ -38,6 +38,7 @@ > #include <asm/pgalloc.h> > #include <asm/sections.h> > #include <asm/setup.h> > +#include <asm/smp.h> > #include <asm/time.h> > > #define SMBIOS_BIOSSIZE_OFFSET 0x09 > @@ -322,6 +323,29 @@ static int __init reserve_memblock_reserved_regions(void) > } > arch_initcall(reserve_memblock_reserved_regions); > > +#ifdef CONFIG_SMP > +static void __init prefill_possible_map(void) > +{ > + int i, possible; > + > + possible = num_processors + disabled_cpus; > + if (possible > nr_cpu_ids) > + possible = nr_cpu_ids; > + > + pr_info("SMP: Allowing %d CPUs, %d hotplug CPUs\n", > + possible, max((possible - num_processors), 0)); > + > + for (i = 0; i < possible; i++) > + set_cpu_possible(i, true); > + for (; i < NR_CPUS; i++) > + set_cpu_possible(i, false); > + > + nr_cpu_ids = possible; > +} > +#else > +static inline void prefill_possible_map(void) {} > +#endif > + > void __init setup_arch(char **cmdline_p) > { > cpu_probe(); > @@ -336,6 +360,8 @@ void __init setup_arch(char **cmdline_p) > arch_mem_init(cmdline_p); > > resource_init(); > + plat_smp_setup(); > + prefill_possible_map(); > > paging_init(); > } > diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c > new file mode 100644 > index 000000000000..99ba7a56edf9 > --- /dev/null > +++ b/arch/loongarch/kernel/smp.c > @@ -0,0 +1,735 @@ > +// SPDX-License-Identifier: GPL-2.0-or-later > +/* > + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited > + * > + * Derived from MIPS: > + * Copyright (C) 2000, 2001 Kanoj Sarcar > + * Copyright (C) 2000, 2001 Ralf Baechle > + * Copyright (C) 2000, 2001 Silicon Graphics, Inc. > + * Copyright (C) 2000, 2001, 2003 Broadcom Corporation > + */ > +#include <linux/cpu.h> > +#include <linux/cpumask.h> > +#include <linux/init.h> > +#include <linux/interrupt.h> > +#include <linux/seq_file.h> > +#include <linux/smp.h> > +#include <linux/threads.h> > +#include <linux/export.h> > +#include <linux/time.h> > +#include <linux/tracepoint.h> > +#include <linux/sched/hotplug.h> > +#include <linux/sched/task_stack.h> > + > +#include <asm/cpu.h> > +#include <asm/idle.h> > +#include <asm/loongson.h> > +#include <asm/mmu_context.h> > +#include <asm/processor.h> > +#include <asm/setup.h> > +#include <asm/time.h> > + > +int __cpu_number_map[NR_CPUS]; /* Map physical to logical */ > +EXPORT_SYMBOL(__cpu_number_map); > + > +int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */ > +EXPORT_SYMBOL(__cpu_logical_map); > + > +/* Number of threads (siblings) per CPU core */ > +int smp_num_siblings = 1; > +EXPORT_SYMBOL(smp_num_siblings); > + > +/* Representing the threads (siblings) of each logical CPU */ > +cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly; > +EXPORT_SYMBOL(cpu_sibling_map); > + > +/* Representing the core map of multi-core chips of each logical CPU */ > +cpumask_t cpu_core_map[NR_CPUS] __read_mostly; > +EXPORT_SYMBOL(cpu_core_map); > + > +static DECLARE_COMPLETION(cpu_starting); > +static DECLARE_COMPLETION(cpu_running); > + > +/* > + * A logcal cpu mask containing only one VPE per core to > + * reduce the number of IPIs on large MT systems. > + */ > +cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly; > +EXPORT_SYMBOL(cpu_foreign_map); > + > +/* representing cpus for which sibling maps can be computed */ > +static cpumask_t cpu_sibling_setup_map; > + > +/* representing cpus for which core maps can be computed */ > +static cpumask_t cpu_core_setup_map; > + > +struct secondary_data cpuboot_data; > +static DEFINE_PER_CPU(int, cpu_state); > +DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); > +EXPORT_PER_CPU_SYMBOL(irq_stat); > + > +enum ipi_msg_type { > + IPI_RESCHEDULE, > + IPI_CALL_FUNCTION, > +}; > + > +static const char *ipi_types[NR_IPI] __tracepoint_string = { > + [IPI_RESCHEDULE] = "Rescheduling interrupts", > + [IPI_CALL_FUNCTION] = "Function call interrupts", > +}; > + > +void show_ipi_list(struct seq_file *p, int prec) > +{ > + unsigned int cpu, i; > + > + for (i = 0; i < NR_IPI; i++) { > + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : ""); > + for_each_online_cpu(cpu) > + seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]); > + seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]); > + } > +} > + > +/* Send mailbox buffer via Mail_Send */ > +static void csr_mail_send(uint64_t data, int cpu, int mailbox) > +{ > + uint64_t val; > + > + /* Send high 32 bits */ > + val = IOCSR_MBUF_SEND_BLOCKING; > + val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); > + val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); > + val |= (data & IOCSR_MBUF_SEND_H32_MASK); > + iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); > + > + /* Send low 32 bits */ > + val = IOCSR_MBUF_SEND_BLOCKING; > + val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT); > + val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT); > + val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT); > + iocsr_write64(val, LOONGARCH_IOCSR_MBUF_SEND); > +}; > + > +static u32 ipi_read_clear(int cpu) > +{ > + u32 action; > + > + /* Load the ipi register to figure out what we're supposed to do */ > + action = iocsr_read32(LOONGARCH_IOCSR_IPI_STATUS); > + /* Clear the ipi register to clear the interrupt */ > + iocsr_write32(action, LOONGARCH_IOCSR_IPI_CLEAR); > + smp_mb(); > + > + return action; > +} > + > +static void ipi_write_action(int cpu, u32 action) > +{ > + unsigned int irq = 0; > + > + while ((irq = ffs(action))) { > + uint32_t val = IOCSR_IPI_SEND_BLOCKING; > + > + val |= (irq - 1); > + val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT); > + iocsr_write32(val, LOONGARCH_IOCSR_IPI_SEND); > + action &= ~BIT(irq - 1); > + } > +} > + > +void loongson3_send_ipi_single(int cpu, unsigned int action) > +{ > + ipi_write_action(cpu_logical_map(cpu), (u32)action); > +} > + > +void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int > action) > +{ > + unsigned int i; > + > + for_each_cpu(i, mask) > + ipi_write_action(cpu_logical_map(i), (u32)action); > +} > + > +irqreturn_t loongson3_ipi_interrupt(int irq, void *dev) > +{ > + unsigned int action; > + unsigned int cpu = smp_processor_id(); > + > + action = ipi_read_clear(cpu_logical_map(cpu)); > + > + if (action & SMP_RESCHEDULE) { > + scheduler_ipi(); > + per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++; > + } > + > + if (action & SMP_CALL_FUNCTION) { > + generic_smp_call_function_interrupt(); > + per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++; > + } > + > + return IRQ_HANDLED; > +} > + > +void __init loongson3_smp_setup(void) > +{ > + cpu_data[0].core = cpu_logical_map(0) % > loongson_sysconf.cores_per_package; > + cpu_data[0].package = cpu_logical_map(0) / > loongson_sysconf.cores_per_package; > + > + iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); > + pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus); > +} > + > +void __init loongson3_prepare_cpus(unsigned int max_cpus) > +{ > + int i = 0; > + > + for (i = 0; i < loongson_sysconf.nr_cpus; i++) { > + set_cpu_present(i, true); > + csr_mail_send(0, __cpu_logical_map[i], 0); > + } > + > + per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; > +} > + > +/* > + * Setup the PC, SP, and TP of a secondary processor and start it > running! > + */ > +void loongson3_boot_secondary(int cpu, struct task_struct *idle) > +{ > + unsigned long entry; > + > + pr_info("Booting CPU#%d...\n", cpu); > + > + entry = __pa_symbol((unsigned long)&smpboot_entry); > + cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle); > + cpuboot_data.thread_info = (unsigned long)task_thread_info(idle); > + > + csr_mail_send(entry, cpu_logical_map(cpu), 0); > + > + loongson3_send_ipi_single(cpu, SMP_BOOT_CPU); > +} > + > +/* > + * SMP init and finish on secondary CPUs > + */ > +void loongson3_init_secondary(void) > +{ > + unsigned int cpu = smp_processor_id(); > + unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 | > + ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER; > + > + change_csr_ecfg(ECFG0_IM, imask); > + > + iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); > + > + per_cpu(cpu_state, cpu) = CPU_ONLINE; > + cpu_data[cpu].core = > + cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; > + cpu_data[cpu].package = > + cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; > +} > + > +void loongson3_smp_finish(void) > +{ > + local_irq_enable(); > + iocsr_write64(0, LOONGARCH_IOCSR_MBUF0); > + pr_info("CPU#%d finished\n", smp_processor_id()); > +} > + > +#ifdef CONFIG_HOTPLUG_CPU > + > +static bool io_master(int cpu) > +{ > + if (cpu == 0) > + return true; > + > + return false; > +} > + > +int loongson3_cpu_disable(void) > +{ > + unsigned long flags; > + unsigned int cpu = smp_processor_id(); > + > + if (io_master(cpu)) > + return -EBUSY; > + > + set_cpu_online(cpu, false); > + calculate_cpu_foreign_map(); > + local_irq_save(flags); > + irq_migrate_all_off_this_cpu(); > + clear_csr_ecfg(ECFG0_IM); > + local_irq_restore(flags); > + local_flush_tlb_all(); > + > + return 0; > +} > + > +void loongson3_cpu_die(unsigned int cpu) > +{ > + while (per_cpu(cpu_state, cpu) != CPU_DEAD) > + cpu_relax(); > + > + mb(); > +} > + > +/* > + * The target CPU should go to XKPRANGE (uncached area) and flush > + * ICache/DCache/VCache before the control CPU can safely disable its > clock. > + */ > +static void loongson3_play_dead(int *state_addr) > +{ > + register int val; > + register void *addr; > + register void (*init_fn)(void); > + > + __asm__ __volatile__( > + " li.d %[addr], 0x8000000000000000\n" > + "1: cacop 0x8, %[addr], 0 \n" /* flush ICache */ > + " cacop 0x8, %[addr], 1 \n" > + " cacop 0x8, %[addr], 2 \n" > + " cacop 0x8, %[addr], 3 \n" > + " cacop 0x9, %[addr], 0 \n" /* flush DCache */ > + " cacop 0x9, %[addr], 1 \n" > + " cacop 0x9, %[addr], 2 \n" > + " cacop 0x9, %[addr], 3 \n" > + " addi.w %[sets], %[sets], -1 \n" > + " addi.d %[addr], %[addr], 0x40 \n" > + " bnez %[sets], 1b \n" > + " li.d %[addr], 0x8000000000000000\n" > + "2: cacop 0xa, %[addr], 0 \n" /* flush VCache */ > + " cacop 0xa, %[addr], 1 \n" > + " cacop 0xa, %[addr], 2 \n" > + " cacop 0xa, %[addr], 3 \n" > + " cacop 0xa, %[addr], 4 \n" > + " cacop 0xa, %[addr], 5 \n" > + " cacop 0xa, %[addr], 6 \n" > + " cacop 0xa, %[addr], 7 \n" > + " cacop 0xa, %[addr], 8 \n" > + " cacop 0xa, %[addr], 9 \n" > + " cacop 0xa, %[addr], 10 \n" > + " cacop 0xa, %[addr], 11 \n" > + " cacop 0xa, %[addr], 12 \n" > + " cacop 0xa, %[addr], 13 \n" > + " cacop 0xa, %[addr], 14 \n" > + " cacop 0xa, %[addr], 15 \n" > + " addi.w %[vsets], %[vsets], -1 \n" > + " addi.d %[addr], %[addr], 0x40 \n" > + " bnez %[vsets], 2b \n" > + " li.w %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */ > + " st.w %[val], %[state_addr], 0 \n" > + " dbar 0 \n" > + " cacop 0x11, %[state_addr], 0 \n" /* flush entry of > *state_addr */ > + : [addr] "=&r" (addr), [val] "=&r" (val) > + : [state_addr] "r" (state_addr), > + [sets] "r" (cpu_data[smp_processor_id()].dcache.sets), > + [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets)); > + > + local_irq_enable(); > + change_csr_ecfg(ECFG0_IM, ECFGF_IPI); > + > + __asm__ __volatile__( > + " idle 0 \n" > + " li.w $t0, 0x1020 \n" > + " iocsrrd.d %[init_fn], $t0 \n" /* Get init PC */ > + : [init_fn] "=&r" (addr) > + : /* No Input */ > + : "a0"); > + init_fn = __va(addr); > + > + init_fn(); > + unreachable(); > +} > + > +void play_dead(void) > +{ > + int *state_addr; > + unsigned int cpu = smp_processor_id(); > + void (*play_dead_uncached)(int *s); > + > + idle_task_exit(); > + play_dead_uncached = (void *)TO_UNCACHE(__pa((unsigned > long)loongson3_play_dead)); > + state_addr = &per_cpu(cpu_state, cpu); > + mb(); > + play_dead_uncached(state_addr); > +} > + > +static int loongson3_enable_clock(unsigned int cpu) > +{ > + uint64_t core_id = cpu_data[cpu].core; > + uint64_t package_id = cpu_data[cpu].package; > + > + LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3); > + > + return 0; > +} > + > +static int loongson3_disable_clock(unsigned int cpu) > +{ > + uint64_t core_id = cpu_data[cpu].core; > + uint64_t package_id = cpu_data[cpu].package; > + > + LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3)); > + > + return 0; > +} > + > +static int register_loongson3_notifier(void) > +{ > + return cpuhp_setup_state_nocalls(CPUHP_LOONGARCH_SOC_PREPARE, > + "loongarch/loongson:prepare", > + loongson3_enable_clock, > + loongson3_disable_clock); > +} > +early_initcall(register_loongson3_notifier); > + > +#endif > + > +/* > + * Power management > + */ > +#ifdef CONFIG_PM > + > +static int loongson3_ipi_suspend(void) > +{ > + return 0; > +} > + > +static void loongson3_ipi_resume(void) > +{ > + iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); > +} > + > +static struct syscore_ops loongson3_ipi_syscore_ops = { > + .resume = loongson3_ipi_resume, > + .suspend = loongson3_ipi_suspend, > +}; > + > +/* > + * Enable boot cpu ipi before enabling nonboot cpus > + * during syscore_resume. > + */ > +static int __init ipi_pm_init(void) > +{ > + register_syscore_ops(&loongson3_ipi_syscore_ops); > + return 0; > +} > + > +core_initcall(ipi_pm_init); > +#endif > + > +static inline void set_cpu_sibling_map(int cpu) > +{ > + int i; > + > + cpumask_set_cpu(cpu, &cpu_sibling_setup_map); > + > + if (smp_num_siblings <= 1) > + cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]); > + else { > + for_each_cpu(i, &cpu_sibling_setup_map) { > + if (cpus_are_siblings(cpu, i)) { > + cpumask_set_cpu(i, &cpu_sibling_map[cpu]); > + cpumask_set_cpu(cpu, &cpu_sibling_map[i]); > + } > + } > + } > +} > + > +static inline void set_cpu_core_map(int cpu) > +{ > + int i; > + > + cpumask_set_cpu(cpu, &cpu_core_setup_map); > + > + for_each_cpu(i, &cpu_core_setup_map) { > + if (cpu_data[cpu].package == cpu_data[i].package) { > + cpumask_set_cpu(i, &cpu_core_map[cpu]); > + cpumask_set_cpu(cpu, &cpu_core_map[i]); > + } > + } > +} > + > +/* > + * Calculate a new cpu_foreign_map mask whenever a > + * new cpu appears or disappears. > + */ > +void calculate_cpu_foreign_map(void) > +{ > + int i, k, core_present; > + cpumask_t temp_foreign_map; > + > + /* Re-calculate the mask */ > + cpumask_clear(&temp_foreign_map); > + for_each_online_cpu(i) { > + core_present = 0; > + for_each_cpu(k, &temp_foreign_map) > + if (cpus_are_siblings(i, k)) > + core_present = 1; > + if (!core_present) > + cpumask_set_cpu(i, &temp_foreign_map); > + } > + > + for_each_online_cpu(i) > + cpumask_andnot(&cpu_foreign_map[i], > + &temp_foreign_map, &cpu_sibling_map[i]); > +} > + > +/* Preload SMP state for boot cpu */ > +void smp_prepare_boot_cpu(void) > +{ > + unsigned int cpu; > + > + set_cpu_possible(0, true); > + set_cpu_online(0, true); > + set_my_cpu_offset(per_cpu_offset(0)); > + > + for_each_possible_cpu(cpu) > + set_cpu_numa_node(cpu, 0); > +} > + > +/* called from main before smp_init() */ > +void __init smp_prepare_cpus(unsigned int max_cpus) > +{ > + init_new_context(current, &init_mm); > + current_thread_info()->cpu = 0; > + loongson3_prepare_cpus(max_cpus); > + set_cpu_sibling_map(0); > + set_cpu_core_map(0); > + calculate_cpu_foreign_map(); > +#ifndef CONFIG_HOTPLUG_CPU > + init_cpu_present(cpu_possible_mask); > +#endif > +} > + > +int __cpu_up(unsigned int cpu, struct task_struct *tidle) > +{ > + loongson3_boot_secondary(cpu, tidle); > + > + /* Wait for CPU to start and be ready to sync counters */ > + if (!wait_for_completion_timeout(&cpu_starting, > + msecs_to_jiffies(5000))) { > + pr_crit("CPU%u: failed to start\n", cpu); > + return -EIO; > + } > + > + /* Wait for CPU to finish startup & mark itself online before return > */ > + wait_for_completion(&cpu_running); > + > + return 0; > +} > + > +/* > + * First C code run on the secondary CPUs after being started up by > + * the master. > + */ > +asmlinkage void start_secondary(void) > +{ > + unsigned int cpu; > + > + sync_counter(); > + cpu = smp_processor_id(); > + set_my_cpu_offset(per_cpu_offset(cpu)); > + > + cpu_probe(); > + constant_clockevent_init(); > + loongson3_init_secondary(); > + > + set_cpu_sibling_map(cpu); > + set_cpu_core_map(cpu); > + > + notify_cpu_starting(cpu); > + > + /* Notify boot CPU that we're starting */ > + complete(&cpu_starting); > + > + /* The CPU is running, now mark it online */ > + set_cpu_online(cpu, true); > + > + calculate_cpu_foreign_map(); > + > + /* > + * Notify boot CPU that we're up & online and it can safely return > + * from __cpu_up() > + */ > + complete(&cpu_running); > + > + /* > + * irq will be enabled in loongson3_smp_finish(), enabling it too > + * early is dangerous. > + */ > + WARN_ON_ONCE(!irqs_disabled()); > + loongson3_smp_finish(); > + > + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); > +} > + > +void __init smp_cpus_done(unsigned int max_cpus) > +{ > +} > + > +static void stop_this_cpu(void *dummy) > +{ > + set_cpu_online(smp_processor_id(), false); > + calculate_cpu_foreign_map(); > + local_irq_disable(); > + while (true); > +} > + > +void smp_send_stop(void) > +{ > + smp_call_function(stop_this_cpu, NULL, 0); > +} > + > +int setup_profiling_timer(unsigned int multiplier) > +{ > + return 0; > +} > + > +static void flush_tlb_all_ipi(void *info) > +{ > + local_flush_tlb_all(); > +} > + > +void flush_tlb_all(void) > +{ > + on_each_cpu(flush_tlb_all_ipi, NULL, 1); > +} > + > +static void flush_tlb_mm_ipi(void *mm) > +{ > + local_flush_tlb_mm((struct mm_struct *)mm); > +} > + > +void flush_tlb_mm(struct mm_struct *mm) > +{ > + if (atomic_read(&mm->mm_users) == 0) > + return; /* happens as a result of exit_mmap() */ > + > + preempt_disable(); > + > + if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { > + on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1); > + } else { > + unsigned int cpu; > + > + for_each_online_cpu(cpu) { > + if (cpu != smp_processor_id() && cpu_context(cpu, mm)) > + cpu_context(cpu, mm) = 0; > + } > + local_flush_tlb_mm(mm); > + } > + > + preempt_enable(); > +} > + > +struct flush_tlb_data { > + struct vm_area_struct *vma; > + unsigned long addr1; > + unsigned long addr2; > +}; > + > +static void flush_tlb_range_ipi(void *info) > +{ > + struct flush_tlb_data *fd = info; > + > + local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); > +} > + > +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, > unsigned long end) > +{ > + struct mm_struct *mm = vma->vm_mm; > + > + preempt_disable(); > + if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { > + struct flush_tlb_data fd = { > + .vma = vma, > + .addr1 = start, > + .addr2 = end, > + }; > + > + on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1); > + } else { > + unsigned int cpu; > + int exec = vma->vm_flags & VM_EXEC; > + > + for_each_online_cpu(cpu) { > + /* > + * flush_cache_range() will only fully flush icache if > + * the VMA is executable, otherwise we must invalidate > + * ASID without it appearing to has_valid_asid() as if > + * mm has been completely unused by that CPU. > + */ > + if (cpu != smp_processor_id() && cpu_context(cpu, mm)) > + cpu_context(cpu, mm) = !exec; > + } > + local_flush_tlb_range(vma, start, end); > + } > + preempt_enable(); > +} > + > +static void flush_tlb_kernel_range_ipi(void *info) > +{ > + struct flush_tlb_data *fd = info; > + > + local_flush_tlb_kernel_range(fd->addr1, fd->addr2); > +} > + > +void flush_tlb_kernel_range(unsigned long start, unsigned long end) > +{ > + struct flush_tlb_data fd = { > + .addr1 = start, > + .addr2 = end, > + }; > + > + on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1); > +} > + > +static void flush_tlb_page_ipi(void *info) > +{ > + struct flush_tlb_data *fd = info; > + > + local_flush_tlb_page(fd->vma, fd->addr1); > +} > + > +void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) > +{ > + preempt_disable(); > + if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != > vma->vm_mm)) { > + struct flush_tlb_data fd = { > + .vma = vma, > + .addr1 = page, > + }; > + > + on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1); > + } else { > + unsigned int cpu; > + > + for_each_online_cpu(cpu) { > + /* > + * flush_cache_page() only does partial flushes, so > + * invalidate ASID without it appearing to > + * has_valid_asid() as if mm has been completely unused > + * by that CPU. > + */ > + if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) > + cpu_context(cpu, vma->vm_mm) = 1; > + } > + local_flush_tlb_page(vma, page); > + } > + preempt_enable(); > +} > +EXPORT_SYMBOL(flush_tlb_page); > + > +static void flush_tlb_one_ipi(void *info) > +{ > + unsigned long vaddr = (unsigned long) info; > + > + local_flush_tlb_one(vaddr); > +} > + > +void flush_tlb_one(unsigned long vaddr) > +{ > + on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1); > +} > +EXPORT_SYMBOL(flush_tlb_one); > diff --git a/arch/loongarch/kernel/topology.c > b/arch/loongarch/kernel/topology.c > index 3b2cbb95875b..ab1a75c0b5a6 100644 > --- a/arch/loongarch/kernel/topology.c > +++ b/arch/loongarch/kernel/topology.c > @@ -1,13 +1,52 @@ > // SPDX-License-Identifier: GPL-2.0 > #include <linux/cpu.h> > +#include <linux/cpumask.h> > #include <linux/init.h> > +#include <linux/node.h> > +#include <linux/nodemask.h> > #include <linux/percpu.h> > > -static struct cpu cpu_device; > +static DEFINE_PER_CPU(struct cpu, cpu_devices); > + > +#ifdef CONFIG_HOTPLUG_CPU > +int arch_register_cpu(int cpu) > +{ > + int ret; > + struct cpu *c = &per_cpu(cpu_devices, cpu); > + > + c->hotpluggable = 1; > + ret = register_cpu(c, cpu); > + if (ret < 0) > + pr_warn("register_cpu %d failed (%d)\n", cpu, ret); > + > + return ret; > +} > +EXPORT_SYMBOL(arch_register_cpu); > + > +void arch_unregister_cpu(int cpu) > +{ > + struct cpu *c = &per_cpu(cpu_devices, cpu); > + > + c->hotpluggable = 0; > + unregister_cpu(c); > +} > +EXPORT_SYMBOL(arch_unregister_cpu); > +#endif > > static int __init topology_init(void) > { > - return register_cpu(&cpu_device, 0); > + int i, ret; > + > + for_each_present_cpu(i) { > + struct cpu *c = &per_cpu(cpu_devices, i); > + > + c->hotpluggable = !!i; > + ret = register_cpu(c, i); > + if (ret < 0) > + pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret); > + } > + > + return 0; > } > > subsys_initcall(topology_init); > diff --git a/arch/loongarch/kernel/vmlinux.lds.S > b/arch/loongarch/kernel/vmlinux.lds.S > index 7da4c4d7c50d..006cbb1bd5c6 100644 > --- a/arch/loongarch/kernel/vmlinux.lds.S > +++ b/arch/loongarch/kernel/vmlinux.lds.S > @@ -73,6 +73,10 @@ SECTIONS > EXIT_DATA > } > > +#ifdef CONFIG_SMP > + PERCPU_SECTION(1 << CONFIG_L1_CACHE_SHIFT) > +#endif > + > .init.bss : { > *(.init.bss) > } > diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S > index bef740710a3b..7eee40271577 100644 > --- a/arch/loongarch/mm/tlbex.S > +++ b/arch/loongarch/mm/tlbex.S > @@ -88,7 +88,14 @@ vmalloc_done_load: > slli.d t0, t0, _PTE_T_LOG2 > add.d t1, ra, t0 > > +#ifdef CONFIG_SMP > +smp_pgtable_change_load: > +#endif > +#ifdef CONFIG_SMP > + ll.d t0, t1, 0 > +#else > ld.d t0, t1, 0 > +#endif > tlbsrch > > srli.d ra, t0, _PAGE_PRESENT_SHIFT > @@ -96,7 +103,12 @@ vmalloc_done_load: > beq ra, $r0, nopage_tlb_load > > ori t0, t0, _PAGE_VALID > +#ifdef CONFIG_SMP > + sc.d t0, t1, 0 > + beq t0, $r0, smp_pgtable_change_load > +#else > st.d t0, t1, 0 > +#endif > ori t1, t1, 8 > xori t1, t1, 8 > ld.d t0, t1, 0 > @@ -120,14 +132,24 @@ vmalloc_load: > * spots a huge page. > */ > tlb_huge_update_load: > +#ifdef CONFIG_SMP > + ll.d t0, t1, 0 > +#else > ld.d t0, t1, 0 > +#endif > srli.d ra, t0, _PAGE_PRESENT_SHIFT > andi ra, ra, 1 > beq ra, $r0, nopage_tlb_load > tlbsrch > > ori t0, t0, _PAGE_VALID > +#ifdef CONFIG_SMP > + sc.d t0, t1, 0 > + beq t0, $r0, tlb_huge_update_load > + ld.d t0, t1, 0 > +#else > st.d t0, t1, 0 > +#endif > addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) > addi.d ra, t1, 0 > csrxchg ra, t1, LOONGARCH_CSR_TLBIDX > @@ -173,6 +195,7 @@ tlb_huge_update_load: > csrxchg t1, t0, LOONGARCH_CSR_TLBIDX > > nopage_tlb_load: > + dbar 0 > csrrd ra, EXCEPTION_KS2 > la.abs t0, tlb_do_page_fault_0 > jirl $r0, t0, 0 > @@ -229,7 +252,14 @@ vmalloc_done_store: > slli.d t0, t0, _PTE_T_LOG2 > add.d t1, ra, t0 > > +#ifdef CONFIG_SMP > +smp_pgtable_change_store: > +#endif > +#ifdef CONFIG_SMP > + ll.d t0, t1, 0 > +#else > ld.d t0, t1, 0 > +#endif > tlbsrch > > srli.d ra, t0, _PAGE_PRESENT_SHIFT > @@ -238,7 +268,12 @@ vmalloc_done_store: > bne ra, $r0, nopage_tlb_store > > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) > +#ifdef CONFIG_SMP > + sc.d t0, t1, 0 > + beq t0, $r0, smp_pgtable_change_store > +#else > st.d t0, t1, 0 > +#endif > > ori t1, t1, 8 > xori t1, t1, 8 > @@ -263,7 +298,11 @@ vmalloc_store: > * spots a huge page. > */ > tlb_huge_update_store: > +#ifdef CONFIG_SMP > + ll.d t0, t1, 0 > +#else > ld.d t0, t1, 0 > +#endif > srli.d ra, t0, _PAGE_PRESENT_SHIFT > andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) > xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) > @@ -272,7 +311,13 @@ tlb_huge_update_store: > tlbsrch > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) > > +#ifdef CONFIG_SMP > + sc.d t0, t1, 0 > + beq t0, $r0, tlb_huge_update_store > + ld.d t0, t1, 0 > +#else > st.d t0, t1, 0 > +#endif > addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16) > addi.d ra, t1, 0 > csrxchg ra, t1, LOONGARCH_CSR_TLBIDX > @@ -318,6 +363,7 @@ tlb_huge_update_store: > csrxchg t1, t0, LOONGARCH_CSR_TLBIDX > > nopage_tlb_store: > + dbar 0 > csrrd ra, EXCEPTION_KS2 > la.abs t0, tlb_do_page_fault_1 > jirl $r0, t0, 0 > @@ -373,7 +419,14 @@ vmalloc_done_modify: > slli.d t0, t0, _PTE_T_LOG2 > add.d t1, ra, t0 > > +#ifdef CONFIG_SMP > +smp_pgtable_change_modify: > +#endif > +#ifdef CONFIG_SMP > + ll.d t0, t1, 0 > +#else > ld.d t0, t1, 0 > +#endif > tlbsrch > > srli.d ra, t0, _PAGE_WRITE_SHIFT > @@ -381,7 +434,12 @@ vmalloc_done_modify: > beq ra, $r0, nopage_tlb_modify > > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) > +#ifdef CONFIG_SMP > + sc.d t0, t1, 0 > + beq t0, $r0, smp_pgtable_change_modify > +#else > st.d t0, t1, 0 > +#endif > ori t1, t1, 8 > xori t1, t1, 8 > ld.d t0, t1, 0 > @@ -405,7 +463,11 @@ vmalloc_modify: > * build_tlbchange_handler_head spots a huge page. > */ > tlb_huge_update_modify: > +#ifdef CONFIG_SMP > + ll.d t0, t1, 0 > +#else > ld.d t0, t1, 0 > +#endif > > srli.d ra, t0, _PAGE_WRITE_SHIFT > andi ra, ra, 1 > @@ -414,7 +476,13 @@ tlb_huge_update_modify: > tlbsrch > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) > > +#ifdef CONFIG_SMP > + sc.d t0, t1, 0 > + beq t0, $r0, tlb_huge_update_modify > + ld.d t0, t1, 0 > +#else > st.d t0, t1, 0 > +#endif > /* > * A huge PTE describes an area the size of the > * configured huge page size. This is twice the > @@ -454,6 +522,7 @@ tlb_huge_update_modify: > csrxchg t1, t0, LOONGARCH_CSR_TLBIDX > > nopage_tlb_modify: > + dbar 0 > csrrd ra, EXCEPTION_KS2 > la.abs t0, tlb_do_page_fault_1 > jirl $r0, t0, 0 > diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h > index b66c5f389159..19f0dbfdd7fe 100644 > --- a/include/linux/cpuhotplug.h > +++ b/include/linux/cpuhotplug.h > @@ -130,6 +130,7 @@ enum cpuhp_state { > CPUHP_ZCOMP_PREPARE, > CPUHP_TIMERS_PREPARE, > CPUHP_MIPS_SOC_PREPARE, > + CPUHP_LOONGARCH_SOC_PREPARE, > CPUHP_BP_PREPARE_DYN, > CPUHP_BP_PREPARE_DYN_END = CPUHP_BP_PREPARE_DYN + 20, > CPUHP_BRINGUP_CPU, > -- > 2.27.0 -- - Jiaxun
