On Thu, Sep 14, 2023 at 4:02 PM Leonardo Bras <leobras@xxxxxxxxxx> wrote:
>
> On Sun, Sep 10, 2023 at 04:29:02AM -0400, guoren@xxxxxxxxxx wrote:
> > From: Guo Ren <guoren@xxxxxxxxxxxxxxxxx>
> >
> > Add a static key controlling whether virt_spin_lock() should be
> > called or not. When running on bare metal set the new key to
> > false.
> >
> > The KVM guests fall back to a Test-and-Set spinlock, because fair
> > locks have horrible lock 'holder' preemption issues. The
> > virt_spin_lock_key would shortcut for the
> > queued_spin_lock_slowpath() function that allow virt_spin_lock to
> > hijack it.
> >
> > Signed-off-by: Guo Ren <guoren@xxxxxxxxxxxxxxxxx>
> > Signed-off-by: Guo Ren <guoren@xxxxxxxxxx>
> > ---
> > .../admin-guide/kernel-parameters.txt | 4 +++
> > arch/riscv/include/asm/sbi.h | 8 +++++
> > arch/riscv/include/asm/spinlock.h | 22 ++++++++++++++
> > arch/riscv/kernel/sbi.c | 2 +-
> > arch/riscv/kernel/setup.c | 30 ++++++++++++++++++-
> > 5 files changed, 64 insertions(+), 2 deletions(-)
> >
> > diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
> > index 61cacb8dfd0e..f75bedc50e00 100644
> > --- a/Documentation/admin-guide/kernel-parameters.txt
> > +++ b/Documentation/admin-guide/kernel-parameters.txt
> > @@ -3927,6 +3927,10 @@
> > no_uaccess_flush
> > [PPC] Don't flush the L1-D cache after accessing user data.
> >
> > + no_virt_spin [RISC-V] Disable virt_spin_lock in KVM guest to use
> > + native_queued_spinlock when the nopvspin option is enabled.
> > + This would help vcpu=pcpu scenarios.
> > +
> > novmcoredd [KNL,KDUMP]
> > Disable device dump. Device dump allows drivers to
> > append dump data to vmcore so you can collect driver
> > diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
> > index 501e06e52078..e0233b3d7a5f 100644
> > --- a/arch/riscv/include/asm/sbi.h
> > +++ b/arch/riscv/include/asm/sbi.h
> > @@ -50,6 +50,13 @@ enum sbi_ext_base_fid {
> > SBI_EXT_BASE_GET_MIMPID,
> > };
> >
> > +enum sbi_ext_base_impl_id {
> > + SBI_EXT_BASE_IMPL_ID_BBL = 0,
> > + SBI_EXT_BASE_IMPL_ID_OPENSBI,
> > + SBI_EXT_BASE_IMPL_ID_XVISOR,
> > + SBI_EXT_BASE_IMPL_ID_KVM,
> > +};
> > +
> > enum sbi_ext_time_fid {
> > SBI_EXT_TIME_SET_TIMER = 0,
> > };
> > @@ -269,6 +276,7 @@ int sbi_console_getchar(void);
> > long sbi_get_mvendorid(void);
> > long sbi_get_marchid(void);
> > long sbi_get_mimpid(void);
> > +long sbi_get_firmware_id(void);
> > void sbi_set_timer(uint64_t stime_value);
> > void sbi_shutdown(void);
> > void sbi_send_ipi(unsigned int cpu);
> > diff --git a/arch/riscv/include/asm/spinlock.h b/arch/riscv/include/asm/spinlock.h
> > index 8ea0fee80652..6b38d6616f14 100644
> > --- a/arch/riscv/include/asm/spinlock.h
> > +++ b/arch/riscv/include/asm/spinlock.h
> > @@ -4,6 +4,28 @@
> > #define __ASM_RISCV_SPINLOCK_H
> >
> > #ifdef CONFIG_QUEUED_SPINLOCKS
> > +/*
> > + * The KVM guests fall back to a Test-and-Set spinlock, because fair locks
> > + * have horrible lock 'holder' preemption issues. The virt_spin_lock_key
> > + * would shortcut for the queued_spin_lock_slowpath() function that allow
> > + * virt_spin_lock to hijack it.
> > + */
> > +DECLARE_STATIC_KEY_TRUE(virt_spin_lock_key);
> > +
> > +#define virt_spin_lock virt_spin_lock
> > +static inline bool virt_spin_lock(struct qspinlock *lock)
> > +{
> > + if (!static_branch_likely(&virt_spin_lock_key))
> > + return false;
> > +
> > + do {
> > + while (atomic_read(&lock->val) != 0)
> > + cpu_relax();
> > + } while (atomic_cmpxchg(&lock->val, 0, _Q_LOCKED_VAL) != 0);
> > +
> > + return true;
> > +}
> > +
> > #define _Q_PENDING_LOOPS (1 << 9)
> > #endif
> >
> > diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c
> > index 88eea3a99ee0..cdd45edc8db4 100644
> > --- a/arch/riscv/kernel/sbi.c
> > +++ b/arch/riscv/kernel/sbi.c
> > @@ -555,7 +555,7 @@ static inline long sbi_get_spec_version(void)
> > return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION);
> > }
> >
> > -static inline long sbi_get_firmware_id(void)
> > +long sbi_get_firmware_id(void)
> > {
> > return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID);
> > }
> > diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
> > index 0f084f037651..c57d15b05160 100644
> > --- a/arch/riscv/kernel/setup.c
> > +++ b/arch/riscv/kernel/setup.c
> > @@ -26,6 +26,7 @@
> > #include <asm/alternative.h>
> > #include <asm/cacheflush.h>
> > #include <asm/cpu_ops.h>
> > +#include <asm/cpufeature.h>
> > #include <asm/early_ioremap.h>
> > #include <asm/pgtable.h>
> > #include <asm/setup.h>
> > @@ -283,16 +284,43 @@ DEFINE_STATIC_KEY_TRUE(combo_qspinlock_key);
> > EXPORT_SYMBOL(combo_qspinlock_key);
> > #endif
> >
> > +#ifdef CONFIG_QUEUED_SPINLOCKS
> > +static bool no_virt_spin_key = false;
>
> I suggest no _key, also there is no need for "= false".
> To be consistent with enable_qspinlock, I also suggest
> adding __ro_after_init:
>
> static bool no_virt_spin __ro_after_init;
okay.
>
>
>
> > +DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
> > +
> > +static int __init no_virt_spin_setup(char *p)
> > +{
> > + no_virt_spin_key = true;
> > +
> > + return 0;
> > +}
> > +early_param("no_virt_spin", no_virt_spin_setup);
> > +
> > +static void __init virt_spin_lock_init(void)
> > +{
> > + if (sbi_get_firmware_id() != SBI_EXT_BASE_IMPL_ID_KVM ||
> > + no_virt_spin_key)
> > + static_branch_disable(&virt_spin_lock_key);
> > + else
> > + pr_info("Enable virt_spin_lock\n");
> > +}
> > +#endif
> > +
>
> A new virt_no_spin kernel parameter was introduced, but without
> CONFIG_QUEUED_SPINLOCKS it will silently fail.
>
> I would suggest an #else clause here with a function to print an error /
> warning message about no_virt_spin being invalid in this scenario.
> It will probably help future debugging.
If CONFIG_QUEUED_SPINLOCKS=n, no_virt_spin should be quiet. The
no_virt_spin is one path of qspinlock.
>
>
> > static void __init riscv_spinlock_init(void)
> > {
> > #ifdef CONFIG_RISCV_COMBO_SPINLOCKS
> > - if (!enable_qspinlock_key) {
> > + if (!enable_qspinlock_key &&
> > + (sbi_get_firmware_id() != SBI_EXT_BASE_IMPL_ID_KVM)) {
> > static_branch_disable(&combo_qspinlock_key);
> > pr_info("Ticket spinlock: enabled\n");
> > } else {
> > pr_info("Queued spinlock: enabled\n");
> > }
> > #endif
> > +
> > +#ifdef CONFIG_QUEUED_SPINLOCKS
> > + virt_spin_lock_init();
> > +#endif
> > }
> >
> > extern void __init init_rt_signal_env(void);
> > --
> > 2.36.1
> >
>
> I am probably missing something out, but it looks to me that this patch is
> causing 2 different changes:
> 1 - Enabling no_virt_spin parameter
> 2 - Disabling queued spinlocks for some firmware_id
>
> Wouldn't be better to split those changes in multiple patches?
> Or am I missing the point on why they need to be together?
>
> Thanks!
> Leo
>
--
Best Regards
Guo Ren
