Dave Martin <Dave.Martin@xxxxxxx> writes: > Manipulating the SVE architectural state, including the vector and > predicate registers, first-fault register and the vector length, > requires the use of dedicated instructions added by SVE. > > This patch adds suitable assembly functions for saving and > restoring the SVE registers and querying the vector length. > Setting of the vector length is done as part of register restore. > > Since people building kernels may not all get an SVE-enabled > toolchain for a while, this patch uses macros that generate > explicit opcodes in place of assembler mnemonics. > > Signed-off-by: Dave Martin <Dave.Martin@xxxxxxx> > --- > arch/arm64/include/asm/fpsimd.h | 5 ++ > arch/arm64/include/asm/fpsimdmacros.h | 137 ++++++++++++++++++++++++++++++++++ > arch/arm64/kernel/entry-fpsimd.S | 17 +++++ > 3 files changed, 159 insertions(+) > > diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h > index 410c481..026a7c7 100644 > --- a/arch/arm64/include/asm/fpsimd.h > +++ b/arch/arm64/include/asm/fpsimd.h > @@ -67,6 +67,11 @@ extern void fpsimd_update_current_state(struct fpsimd_state *state); > > extern void fpsimd_flush_task_state(struct task_struct *target); > > +extern void sve_save_state(void *state, u32 *pfpsr); > +extern void sve_load_state(void const *state, u32 const *pfpsr, > + unsigned long vq_minus_1); > +extern unsigned int sve_get_vl(void); > + > /* For use by EFI runtime services calls only */ > extern void __efi_fpsimd_begin(void); > extern void __efi_fpsimd_end(void); > diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h > index 0f5fdd3..5023064 100644 > --- a/arch/arm64/include/asm/fpsimdmacros.h > +++ b/arch/arm64/include/asm/fpsimdmacros.h > @@ -75,3 +75,140 @@ > ldr w\tmpnr, [\state, #16 * 2 + 4] > fpsimd_restore_fpcr x\tmpnr, \state > .endm > + > +.macro _check_general_reg nr > + .if (\nr) < 0 || (\nr) > 30 > + .error "Bad register number \nr." > + .endif > +.endm > + > +.macro _sve_check_zreg znr > + .if (\znr) < 0 || (\znr) > 31 > + .error "Bad Scalable Vector Extension vector register number \znr." > + .endif > +.endm > + > +.macro _sve_check_preg pnr > + .if (\pnr) < 0 || (\pnr) > 15 > + .error "Bad Scalable Vector Extension predicate register number \pnr." > + .endif > +.endm > + > +.macro _check_num n, min, max > + .if (\n) < (\min) || (\n) > (\max) > + .error "Number \n out of range [\min,\max]" > + .endif > +.endm > + > +.macro _sve_str_v nz, nxbase, offset=0 > + _sve_check_zreg \nz > + _check_general_reg \nxbase > + _check_num (\offset), -0x100, 0xff > + .inst 0xe5804000 \ > + | (\nz) \ > + | ((\nxbase) << 5) \ > + | (((\offset) & 7) << 10) \ > + | (((\offset) & 0x1f8) << 13) > +.endm Can we have references to the ARM ARM titles (the section numbers keep changing) for each hand-hacked instruction please. Otherwise it is impossible to verify each one. > + > +.macro _sve_ldr_v nz, nxbase, offset=0 > + _sve_check_zreg \nz > + _check_general_reg \nxbase > + _check_num (\offset), -0x100, 0xff > + .inst 0x85804000 \ > + | (\nz) \ > + | ((\nxbase) << 5) \ > + | (((\offset) & 7) << 10) \ > + | (((\offset) & 0x1f8) << 13) > +.endm > + > +.macro _sve_str_p np, nxbase, offset=0 > + _sve_check_preg \np > + _check_general_reg \nxbase > + _check_num (\offset), -0x100, 0xff > + .inst 0xe5800000 \ > + | (\np) \ > + | ((\nxbase) << 5) \ > + | (((\offset) & 7) << 10) \ > + | (((\offset) & 0x1f8) << 13) > +.endm > + > +.macro _sve_ldr_p np, nxbase, offset=0 > + _sve_check_preg \np > + _check_general_reg \nxbase > + _check_num (\offset), -0x100, 0xff > + .inst 0x85800000 \ > + | (\np) \ > + | ((\nxbase) << 5) \ > + | (((\offset) & 7) << 10) \ > + | (((\offset) & 0x1f8) << 13) > +.endm > + > +.macro _sve_rdvl nx, imm > + _check_general_reg \nx > + _check_num (\imm), -0x20, 0x1f > + .inst 0x04bf5000 \ > + | (\nx) \ > + | (((\imm) & 0x3f) << 5) > +.endm > + > +.macro _sve_rdffr np > + _sve_check_preg \np > + .inst 0x2519f000 \ > + | (\np) > +.endm > + > +.macro _sve_wrffr np > + _sve_check_preg \np > + .inst 0x25289000 \ > + | ((\np) << 5) > +.endm > + > +.macro __for from:req, to:req > + .if (\from) == (\to) > + _for__body \from > + .else > + __for \from, (\from) + ((\to) - (\from)) / 2 > + __for (\from) + ((\to) - (\from)) / 2 + 1, \to > + .endif > +.endm > + > +.macro _for var:req, from:req, to:req, insn:vararg > + .macro _for__body \var:req > + \insn > + .endm > + > + __for \from, \to > + > + .purgem _for__body > +.endm > + > +.macro sve_save nxbase, xpfpsr, nxtmp > + _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 > + _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 > + _sve_rdffr 0 > + _sve_str_p 0, \nxbase > + _sve_ldr_p 0, \nxbase, -16 > + > + mrs x\nxtmp, fpsr > + str w\nxtmp, [\xpfpsr] > + mrs x\nxtmp, fpcr > + str w\nxtmp, [\xpfpsr, #4] > +.endm > + > +.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp > + mrs_s x\nxtmp, SYS_ZCR_EL1 > + bic x\nxtmp, x\nxtmp, ZCR_ELx_LEN_MASK > + orr x\nxtmp, x\nxtmp, \xvqminus1 > + msr_s SYS_ZCR_EL1, x\nxtmp // self-synchronising > + > + _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 > + _sve_ldr_p 0, \nxbase > + _sve_wrffr 0 > + _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 > + > + ldr w\nxtmp, [\xpfpsr] > + msr fpsr, x\nxtmp > + ldr w\nxtmp, [\xpfpsr, #4] > + msr fpcr, x\nxtmp > +.endm > diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S > index 6a27cd6..73f17bf 100644 > --- a/arch/arm64/kernel/entry-fpsimd.S > +++ b/arch/arm64/kernel/entry-fpsimd.S > @@ -41,3 +41,20 @@ ENTRY(fpsimd_load_state) > fpsimd_restore x0, 8 > ret > ENDPROC(fpsimd_load_state) > + > +#ifdef CONFIG_ARM64_SVE > +ENTRY(sve_save_state) > + sve_save 0, x1, 2 > + ret > +ENDPROC(sve_save_state) > + > +ENTRY(sve_load_state) > + sve_load 0, x1, x2, 3 > + ret > +ENDPROC(sve_load_state) > + > +ENTRY(sve_get_vl) > + _sve_rdvl 0, 1 > + ret > +ENDPROC(sve_get_vl) > +#endif /* CONFIG_ARM64_SVE */ -- Alex Bennée _______________________________________________ kvmarm mailing list kvmarm@xxxxxxxxxxxxxxxxxxxxx https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
