On Fri, Jun 21, 2024 at 2:52 AM Evan Green <evan@xxxxxxxxxxxx> wrote: > > On Thu, Jun 13, 2024 at 12:17 PM Jesse Taube <jesse@xxxxxxxxxxxx> wrote: > > > > Run a unaligned vector access to test if the system supports > > vector unaligned access. Add the result to a new key in hwprobe. > > This is useful for usermode to know if vector misaligned accesses are > > supported and if they are faster or slower than equivalent byte accesses. > > > > Signed-off-by: Jesse Taube <jesse@xxxxxxxxxxxx> > > --- > > V1 -> V2: > > - Add Kconfig options > > - Add insn_is_vector > > - Add handle_vector_misaligned_load > > - Fix build > > - Seperate vector from scalar misaligned access > > - This patch was almost completely rewritten > > --- > > arch/riscv/Kconfig | 41 +++++++ > > arch/riscv/include/asm/cpufeature.h | 7 +- > > arch/riscv/include/asm/entry-common.h | 11 -- > > arch/riscv/include/asm/hwprobe.h | 2 +- > > arch/riscv/include/asm/vector.h | 1 + > > arch/riscv/include/uapi/asm/hwprobe.h | 5 + > > arch/riscv/kernel/Makefile | 4 +- > > arch/riscv/kernel/sys_hwprobe.c | 41 +++++++ > > arch/riscv/kernel/traps_misaligned.c | 119 ++++++++++++++++++++- > > arch/riscv/kernel/unaligned_access_speed.c | 9 +- > > arch/riscv/kernel/vector.c | 2 +- > > 11 files changed, 221 insertions(+), 21 deletions(-) > > > > diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig > > index b94176e25be1..f12df0ca6c18 100644 > > --- a/arch/riscv/Kconfig > > +++ b/arch/riscv/Kconfig > > @@ -723,6 +723,12 @@ config RISCV_MISALIGNED > > help > > Embed support for emulating misaligned loads and stores. > > > > +config RISCV_VECTOR_MISALIGNED > > + bool > > + depends on RISCV_ISA_V > > + help > > + Enable detecting support for vector misaligned loads and stores. > > + > > choice > > prompt "Unaligned Accesses Support" > > default RISCV_PROBE_UNALIGNED_ACCESS > > @@ -774,6 +780,41 @@ config RISCV_EFFICIENT_UNALIGNED_ACCESS > > > > endchoice > > > > +choice > > + prompt "Vector unaligned Accesses Support" > > + depends on RISCV_ISA_V > > + default RISCV_PROBE_VECTOR_UNALIGNED_ACCESS > > + help > > + This determines the level of support for vector unaligned accesses. This > > + information is used by the kernel to perform optimizations. It is also > > + exposed to user space via the hwprobe syscall. The hardware will be > > + probed at boot by default. > > + > > +config RISCV_DETECT_VECTOR_UNALIGNED_ACCESS > > + bool "Detect support for vector unaligned accesses" > > + select RISCV_VECTOR_MISALIGNED > > + help > > + During boot, the kernel will detect if the system supports vector > > + unaligned accesses. > > + > > +config RISCV_PROBE_VECTOR_UNALIGNED_ACCESS > > + bool "Probe speed of vector unaligned accesses" > > + select RISCV_VECTOR_MISALIGNED > > + help > > + During boot, the kernel will run a series of tests to determine the > > + speed of vector unaligned accesses if they are supported. This probing > > + will dynamically determine the speed of vector unaligned accesses on > > + the underlying system if they are supported. > > + > > +config CONFIG_RISCV_UNALIGNED_ACCESS_UNSUPPORTED > > + bool "Assume the system does not support vector unaligned memory accesses" > > + help > > + Assume that the system does not support vector unaligned memory accesses. > > + The kernel and userspace programs may run them successfully on systems > > + that do support vector unaligned memory accesses. > > + > > +endchoice > > + > > endmenu # "Platform type" > > > > menu "Kernel features" > > diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h > > index 347805446151..d0ea5921ab20 100644 > > --- a/arch/riscv/include/asm/cpufeature.h > > +++ b/arch/riscv/include/asm/cpufeature.h > > @@ -33,8 +33,8 @@ extern struct riscv_isainfo hart_isa[NR_CPUS]; > > > > void riscv_user_isa_enable(void); > > > > -#if defined(CONFIG_RISCV_MISALIGNED) > > bool check_unaligned_access_emulated_all_cpus(void); > > +#if defined(CONFIG_RISCV_MISALIGNED) > > void unaligned_emulation_finish(void); > > bool unaligned_ctl_available(void); > > DECLARE_PER_CPU(long, misaligned_access_speed); > > @@ -45,6 +45,11 @@ static inline bool unaligned_ctl_available(void) > > } > > #endif > > > > +bool check_vector_unaligned_access_emulated_all_cpus(void); > > +#if defined(CONFIG_RISCV_VECTOR_MISALIGNED) > > +DECLARE_PER_CPU(long, vector_misaligned_access); > > +#endif > > + > > #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) > > DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key); > > > > diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h > > index 2293e535f865..7b32d2b08bb6 100644 > > --- a/arch/riscv/include/asm/entry-common.h > > +++ b/arch/riscv/include/asm/entry-common.h > > @@ -25,18 +25,7 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs, > > void handle_page_fault(struct pt_regs *regs); > > void handle_break(struct pt_regs *regs); > > > > -#ifdef CONFIG_RISCV_MISALIGNED > > int handle_misaligned_load(struct pt_regs *regs); > > int handle_misaligned_store(struct pt_regs *regs); > > -#else > > -static inline int handle_misaligned_load(struct pt_regs *regs) > > -{ > > - return -1; > > -} > > -static inline int handle_misaligned_store(struct pt_regs *regs) > > -{ > > - return -1; > > -} > > -#endif > > > > #endif /* _ASM_RISCV_ENTRY_COMMON_H */ > > diff --git a/arch/riscv/include/asm/hwprobe.h b/arch/riscv/include/asm/hwprobe.h > > index 150a9877b0af..ef01c182af2b 100644 > > --- a/arch/riscv/include/asm/hwprobe.h > > +++ b/arch/riscv/include/asm/hwprobe.h > > @@ -8,7 +8,7 @@ > > > > #include <uapi/asm/hwprobe.h> > > > > -#define RISCV_HWPROBE_MAX_KEY 7 > > +#define RISCV_HWPROBE_MAX_KEY 8 > > > > static inline bool riscv_hwprobe_key_is_valid(__s64 key) > > { > > diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h > > index be7d309cca8a..99b0f91db9ee 100644 > > --- a/arch/riscv/include/asm/vector.h > > +++ b/arch/riscv/include/asm/vector.h > > @@ -21,6 +21,7 @@ > > > > extern unsigned long riscv_v_vsize; > > int riscv_v_setup_vsize(void); > > +bool insn_is_vector(u32 insn_buf); > > bool riscv_v_first_use_handler(struct pt_regs *regs); > > void kernel_vector_begin(void); > > void kernel_vector_end(void); > > diff --git a/arch/riscv/include/uapi/asm/hwprobe.h b/arch/riscv/include/uapi/asm/hwprobe.h > > index 023b7771d1b7..2fee870e41bb 100644 > > --- a/arch/riscv/include/uapi/asm/hwprobe.h > > +++ b/arch/riscv/include/uapi/asm/hwprobe.h > > @@ -75,6 +75,11 @@ struct riscv_hwprobe { > > #define RISCV_HWPROBE_MISALIGNED_MASK (7 << 0) > > #define RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE 6 > > #define RISCV_HWPROBE_KEY_MISALIGNED_PERF 7 > > +#define RISCV_HWPROBE_KEY_VEC_MISALIGNED_PERF 8 > > +#define RISCV_HWPROBE_VEC_MISALIGNED_UNKNOWN 0 > > +#define RISCV_HWPROBE_VEC_MISALIGNED_SLOW 2 > > +#define RISCV_HWPROBE_VEC_MISALIGNED_FAST 3 > > +#define RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED 4 > > /* Increase RISCV_HWPROBE_MAX_KEY when adding items. */ > > > > /* Flags */ > > diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile > > index 5b243d46f4b1..62ac19c029f1 100644 > > --- a/arch/riscv/kernel/Makefile > > +++ b/arch/riscv/kernel/Makefile > > @@ -62,8 +62,8 @@ obj-y += probes/ > > obj-y += tests/ > > obj-$(CONFIG_MMU) += vdso.o vdso/ > > > > -obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o > > -obj-$(CONFIG_RISCV_MISALIGNED) += unaligned_access_speed.o > > +obj-y += traps_misaligned.o > > +obj-y += unaligned_access_speed.o > > obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) += copy-unaligned.o > > > > obj-$(CONFIG_FPU) += fpu.o > > diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c > > index e910e2971984..c40df314058b 100644 > > --- a/arch/riscv/kernel/sys_hwprobe.c > > +++ b/arch/riscv/kernel/sys_hwprobe.c > > @@ -194,6 +194,43 @@ static u64 hwprobe_misaligned(const struct cpumask *cpus) > > } > > #endif > > > > +#ifdef CONFIG_RISCV_VECTOR_MISALIGNED > > +static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) > > +{ > > + int cpu; > > + u64 perf = -1ULL; > > + > > + if (IS_ENABLED(CONFIG_RISCV_UNALIGNED_ACCESS_UNSUPPORTED)) > > + return RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED; > > + > > + /* Return if supported or not even if speed wasn't probed */ > > + for_each_cpu(cpu, cpus) { > > + int this_perf = per_cpu(vector_misaligned_access, cpu); > > + > > + if (perf == -1ULL) > > + perf = this_perf; > > + > > + if (perf != this_perf) { > > + perf = RISCV_HWPROBE_VEC_MISALIGNED_UNKNOWN; > > + break; > > + } > > + } > > + > > + if (perf == -1ULL) > > + return RISCV_HWPROBE_VEC_MISALIGNED_UNKNOWN; > > + > > + return perf; > > +} > > +#else > > +static u64 hwprobe_vec_misaligned(const struct cpumask *cpus) > > +{ > > + if (IS_ENABLED(CONFIG_RISCV_UNALIGNED_ACCESS_UNSUPPORTED)) > > + return RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED; > > + > > + return RISCV_HWPROBE_VEC_MISALIGNED_UNKNOWN; > > +} > > +#endif > > + > > static void hwprobe_one_pair(struct riscv_hwprobe *pair, > > const struct cpumask *cpus) > > { > > @@ -222,6 +259,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, > > pair->value = hwprobe_misaligned(cpus); > > break; > > > > + case RISCV_HWPROBE_KEY_VEC_MISALIGNED_PERF: > > + pair->value = hwprobe_vec_misaligned(cpus); > > + break; > > + > > case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE: > > pair->value = 0; > > if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ)) > > diff --git a/arch/riscv/kernel/traps_misaligned.c b/arch/riscv/kernel/traps_misaligned.c > > index 8fadbe00dd62..6f0264a8c9de 100644 > > --- a/arch/riscv/kernel/traps_misaligned.c > > +++ b/arch/riscv/kernel/traps_misaligned.c > > @@ -16,6 +16,7 @@ > > #include <asm/entry-common.h> > > #include <asm/hwprobe.h> > > #include <asm/cpufeature.h> > > +#include <asm/vector.h> > > > > #define INSN_MATCH_LB 0x3 > > #define INSN_MASK_LB 0x707f > > @@ -322,12 +323,37 @@ union reg_data { > > u64 data_u64; > > }; > > > > -static bool unaligned_ctl __read_mostly; > > - > > /* sysctl hooks */ > > int unaligned_enabled __read_mostly = 1; /* Enabled by default */ > > > > -int handle_misaligned_load(struct pt_regs *regs) > > +#ifdef CONFIG_RISCV_VECTOR_MISALIGNED > > +static int handle_vector_misaligned_load(struct pt_regs *regs) > > +{ > > + unsigned long epc = regs->epc; > > + unsigned long insn; > > + > > + if (get_insn(regs, epc, &insn)) > > + return -1; > > + > > + /* Only return 0 when in check_vector_unaligned_access_emulated */ > > + if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_VEC_MISALIGNED_UNKNOWN) { > > + *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED; > > + regs->epc = epc + INSN_LEN(insn); > > + return 0; > > + } > > + > > + /* If vector instruction we don't emulate it yet */ > > + regs->epc = epc; > > + return -1; > > +} > > +#else > > +static int handle_vector_misaligned_load(struct pt_regs *regs) > > +{ > > + return -1; > > +} > > +#endif > > + > > +static int handle_scalar_misaligned_load(struct pt_regs *regs) > > { > > union reg_data val; > > unsigned long epc = regs->epc; > > @@ -435,7 +461,7 @@ int handle_misaligned_load(struct pt_regs *regs) > > return 0; > > } > > > > -int handle_misaligned_store(struct pt_regs *regs) > > +static int handle_scalar_misaligned_store(struct pt_regs *regs) > > { > > union reg_data val; > > unsigned long epc = regs->epc; > > @@ -526,6 +552,85 @@ int handle_misaligned_store(struct pt_regs *regs) > > return 0; > > } > > > > +int handle_misaligned_load(struct pt_regs *regs) > > +{ > > + unsigned long epc = regs->epc; > > + unsigned long insn; > > + > > + if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) { > > + if (get_insn(regs, epc, &insn)) > > + return -1; > > + > > + if (insn_is_vector(insn)) > > + return handle_vector_misaligned_load(regs); > > + } > > + > > + if (IS_ENABLED(CONFIG_RISCV_MISALIGNED)) > > + return handle_scalar_misaligned_load(regs); > > + > > + return -1; > > +} > > + > > +int handle_misaligned_store(struct pt_regs *regs) > > +{ > > + if (IS_ENABLED(CONFIG_RISCV_MISALIGNED)) > > + return handle_scalar_misaligned_store(regs); > > + > > + return -1; > > +} > > + > > +#ifdef CONFIG_RISCV_VECTOR_MISALIGNED > > +static void check_vector_unaligned_access_emulated(struct work_struct *unused) > > +{ > > + long *mas_ptr = this_cpu_ptr(&vector_misaligned_access); > > + unsigned long tmp_var; > > + > > + *mas_ptr = RISCV_HWPROBE_VEC_MISALIGNED_UNKNOWN; > > + > > + local_irq_enable(); > > Generally if a function is called with interrupts disabled there's a > reason for it, like the system will crash if an interrupt fires during > execution of this region. I haven't researched this, but to feel > comfortable I'd want to know why interrupts were disabled on entry > here, why it's safe to enable them now, and why it's safe to return > from the function with them still enabled. > > I'm guessing this was added because may_use_simd() was blowing up for > you without it. If that's the case, I think we'll need to reconcile > that in a different way. From a quick glance at kernel_mode_vector.c, > I was originally thinking may_use_simd() enforces this because there's Current nesting support of kernel mode vector only allows softirq context to use v context nesting on top of kernel thread's v context. I did not expect the use of v in the interrupt handler but I am open to discussions if it is needed. The reason why may_use_simd() checks for irq_disabled() is that local_bh_enable later in kernel_vector_end() expects running with irq on. Although the success path of preemptible v does not require !irq_disabled(), the kernel falls back to non-preemptible v (get_cpu_vector_context) in some cases. First is the nesting softirq case, or a rare case where an exception handler uses v in a kernel fault that also uses v. The second is when memory allocation for kernel_vstate fails. The fall back uses the non-preemptible v, so it is linked to bh_*. We can get rid of the last case by failing the launch of a kernel thread when the allocation fails. > only a single instance of current->thread.kernel_vstate. So if you > tried to start a v context from an interrupt, you may land on top of > another kernel user, and there's nowhere to save that kernel user's > old state. But there does seem to be some support for nested V context > associated with CONFIG_RISCV_ISA_V_PREEMPTIVE, so I'm a little > confused. It seems like your options are to try and get running on > each CPU in a different manner (such that you're not stuck in irq > context by the time you first run), or try and dive into the kernel V > context code to enable support for starting a V context in irq land. > > > > + kernel_vector_begin(); > > + __asm__ __volatile__ ( > > + ".balign 4\n\t" > > + ".option push\n\t" > > + ".option arch, +zve32x\n\t" > > + " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b > > + " vle16.v v0, (%[ptr])\n\t" // Load bytes > > + ".option pop\n\t" > > + : : [ptr] "r" ((u8 *)&tmp_var + 1) : "v0"); > > + kernel_vector_end(); > > +} > > + > > +bool check_vector_unaligned_access_emulated_all_cpus(void) > > +{ > > + int cpu; > > + bool ret = true; > > + > > + if (!has_vector()) { > > + for_each_online_cpu(cpu) > > + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED; > > + return false; > > + } > > + > > + schedule_on_each_cpu(check_vector_unaligned_access_emulated); > > + > > + for_each_online_cpu(cpu) > > + if (per_cpu(vector_misaligned_access, cpu) > > + != RISCV_HWPROBE_VEC_MISALIGNED_SLOW) > > + return false; > > + > > + return ret; > > +} > > +#else > > +bool check_vector_unaligned_access_emulated_all_cpus(void) > > +{ > > + return false; > > +} > > +#endif > > + > > +#ifdef CONFIG_RISCV_MISALIGNED > > + > > +static bool unaligned_ctl __read_mostly; > > + > > static void check_unaligned_access_emulated(struct work_struct *unused) > > { > > int cpu = smp_processor_id(); > > @@ -563,3 +668,9 @@ bool unaligned_ctl_available(void) > > { > > return unaligned_ctl; > > } > > +#else > > +bool check_unaligned_access_emulated_all_cpus(void) > > +{ > > + return false; > > +} > > +#endif > > diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c > > index 70c1588fc353..c6106bd4a25a 100644 > > --- a/arch/riscv/kernel/unaligned_access_speed.c > > +++ b/arch/riscv/kernel/unaligned_access_speed.c > > @@ -19,7 +19,8 @@ > > #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE) > > #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) > > > > -DEFINE_PER_CPU(long, misaligned_access_speed); > > +DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_UNKNOWN; > > +DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_VEC_MISALIGNED_UNSUPPORTED; > > > > #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS > > static cpumask_t fast_misaligned_access; > > @@ -268,12 +269,18 @@ static int check_unaligned_access_all_cpus(void) > > > > if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICCLSM)) { > > for_each_online_cpu(cpu) { > > +#ifdef CONFIG_RISCV_VECTOR_MISALIGNED > > + per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_VEC_MISALIGNED_FAST; > > +#endif > > +#ifdef CONFIG_RISCV_MISALIGNED > > per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_FAST; > > +#endif > > } > > return 0; > > } > > > > all_cpus_emulated = check_unaligned_access_emulated_all_cpus(); > > + check_vector_unaligned_access_emulated_all_cpus(); > > > > #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS > > if (!all_cpus_emulated) > > diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c > > index 682b3feee451..821818886fab 100644 > > --- a/arch/riscv/kernel/vector.c > > +++ b/arch/riscv/kernel/vector.c > > @@ -66,7 +66,7 @@ void __init riscv_v_setup_ctx_cache(void) > > #endif > > } > > > > -static bool insn_is_vector(u32 insn_buf) > > +bool insn_is_vector(u32 insn_buf) > > { > > u32 opcode = insn_buf & __INSN_OPCODE_MASK; > > u32 width, csr; > > -- > > 2.43.0 > > Thanks, Andy