From: Guo Ren <guoren@xxxxxxxxxxxxxxxxx>
Combo spinlock could support queued and ticket in one Linux Image and
select them during boot time via errata mechanism. Here is the func
size (Bytes) comparison table below:
TYPE : COMBO | TICKET | QUEUED
arch_spin_lock : 106 | 60 | 50
arch_spin_unlock : 54 | 36 | 26
arch_spin_trylock : 110 | 72 | 54
arch_spin_is_locked : 48 | 34 | 20
arch_spin_is_contended : 56 | 40 | 24
rch_spin_value_unlocked : 48 | 34 | 24
One example of disassemble combo arch_spin_unlock:
0xffffffff8000409c <+14>: nop # detour slot
0xffffffff800040a0 <+18>: fence rw,w # queued spinlock start
0xffffffff800040a4 <+22>: sb zero,0(a4) # queued spinlock end
0xffffffff800040a8 <+26>: ld s0,8(sp)
0xffffffff800040aa <+28>: addi sp,sp,16
0xffffffff800040ac <+30>: ret
0xffffffff800040ae <+32>: lw a5,0(a4) # ticket spinlock start
0xffffffff800040b0 <+34>: sext.w a5,a5
0xffffffff800040b2 <+36>: fence rw,w
0xffffffff800040b6 <+40>: addiw a5,a5,1
0xffffffff800040b8 <+42>: slli a5,a5,0x30
0xffffffff800040ba <+44>: srli a5,a5,0x30
0xffffffff800040bc <+46>: sh a5,0(a4) # ticket spinlock end
0xffffffff800040c0 <+50>: ld s0,8(sp)
0xffffffff800040c2 <+52>: addi sp,sp,16
0xffffffff800040c4 <+54>: ret
The qspinlock is smaller and faster than ticket-lock when all are in
fast-path, and combo spinlock could provide a compatible Linux Image
for different micro-arch design (weak/strict fwd guarantee LR/SC)
processors.
Signed-off-by: Guo Ren <guoren@xxxxxxxxxx>
Signed-off-by: Guo Ren <guoren@xxxxxxxxxxxxxxxxx>
---
arch/riscv/Kconfig | 9 +++-
arch/riscv/include/asm/spinlock.h | 78 ++++++++++++++++++++++++++++++-
arch/riscv/kernel/setup.c | 14 ++++++
3 files changed, 98 insertions(+), 3 deletions(-)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 7f39bfc75744..4bcff2860f48 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -473,7 +473,7 @@ config NODES_SHIFT
choice
prompt "RISC-V spinlock type"
- default RISCV_TICKET_SPINLOCKS
+ default RISCV_COMBO_SPINLOCKS
config RISCV_TICKET_SPINLOCKS
bool "Using ticket spinlock"
@@ -485,6 +485,13 @@ config RISCV_QUEUED_SPINLOCKS
help
Make sure your micro arch LL/SC has a strong forward progress guarantee.
Otherwise, stay at ticket-lock.
+
+config RISCV_COMBO_SPINLOCKS
+ bool "Using combo spinlock"
+ depends on SMP && MMU
+ select ARCH_USE_QUEUED_SPINLOCKS
+ help
+ Select queued spinlock or ticket-lock via errata.
endchoice
config RISCV_ALTERNATIVE
diff --git a/arch/riscv/include/asm/spinlock.h b/arch/riscv/include/asm/spinlock.h
index c644a92d4548..8ea0fee80652 100644
--- a/arch/riscv/include/asm/spinlock.h
+++ b/arch/riscv/include/asm/spinlock.h
@@ -7,11 +7,85 @@
#define _Q_PENDING_LOOPS (1 << 9)
#endif
+#ifdef CONFIG_RISCV_COMBO_SPINLOCKS
+#include <asm-generic/ticket_spinlock.h>
+
+#undef arch_spin_is_locked
+#undef arch_spin_is_contended
+#undef arch_spin_value_unlocked
+#undef arch_spin_lock
+#undef arch_spin_trylock
+#undef arch_spin_unlock
+
+#include <asm-generic/qspinlock.h>
+#include <linux/jump_label.h>
+
+#undef arch_spin_is_locked
+#undef arch_spin_is_contended
+#undef arch_spin_value_unlocked
+#undef arch_spin_lock
+#undef arch_spin_trylock
+#undef arch_spin_unlock
+
+DECLARE_STATIC_KEY_TRUE(combo_qspinlock_key);
+
+static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
+{
+ if (static_branch_likely(&combo_qspinlock_key))
+ queued_spin_lock(lock);
+ else
+ ticket_spin_lock(lock);
+}
+
+static __always_inline bool arch_spin_trylock(arch_spinlock_t *lock)
+{
+ if (static_branch_likely(&combo_qspinlock_key))
+ return queued_spin_trylock(lock);
+ else
+ return ticket_spin_trylock(lock);
+}
+
+static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
+{
+ if (static_branch_likely(&combo_qspinlock_key))
+ queued_spin_unlock(lock);
+ else
+ ticket_spin_unlock(lock);
+}
+
+static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
+{
+ if (static_branch_likely(&combo_qspinlock_key))
+ return queued_spin_value_unlocked(lock);
+ else
+ return ticket_spin_value_unlocked(lock);
+}
+
+static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock)
+{
+ if (static_branch_likely(&combo_qspinlock_key))
+ return queued_spin_is_locked(lock);
+ else
+ return ticket_spin_is_locked(lock);
+}
+
+static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock)
+{
+ if (static_branch_likely(&combo_qspinlock_key))
+ return queued_spin_is_contended(lock);
+ else
+ return ticket_spin_is_contended(lock);
+}
+#else /* CONFIG_RISCV_COMBO_SPINLOCKS */
+
#ifdef CONFIG_QUEUED_SPINLOCKS
#include <asm/qspinlock.h>
-#include <asm/qrwlock.h>
#else
-#include <asm-generic/spinlock.h>
+#include <asm-generic/ticket_spinlock.h>
#endif
+#endif /* CONFIG_RISCV_COMBO_SPINLOCKS */
+
+#include <asm/qrwlock.h>
+
#endif /* __ASM_RISCV_SPINLOCK_H */
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index 32c2e1eb71bd..a447cf360a18 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -269,6 +269,18 @@ static void __init parse_dtb(void)
#endif
}
+#ifdef CONFIG_RISCV_COMBO_SPINLOCKS
+DEFINE_STATIC_KEY_TRUE(combo_qspinlock_key);
+EXPORT_SYMBOL(combo_qspinlock_key);
+#endif
+
+static void __init riscv_spinlock_init(void)
+{
+#ifdef CONFIG_RISCV_COMBO_SPINLOCKS
+ static_branch_disable(&combo_qspinlock_key);
+#endif
+}
+
extern void __init init_rt_signal_env(void);
void __init setup_arch(char **cmdline_p)
@@ -317,6 +329,8 @@ void __init setup_arch(char **cmdline_p)
riscv_isa_extension_available(NULL, ZICBOM))
riscv_noncoherent_supported();
riscv_set_dma_cache_alignment();
+
+ riscv_spinlock_init();
}
static int __init topology_init(void)
--
2.36.1
