From: James Morse <james.morse@xxxxxxx> KVM writes the Spectre-v2 mitigation template at the beginning of each vector when a CPU requires a specific sequence to run. Because the template is copied, it can not be modified by the alternatives at runtime. As the KVM template code is intertwined with the bp-hardening callbacks, all templates must have a bp-hardening callback. Add templates for calling ARCH_WORKAROUND_3 and one for each value of K in the brancy-loop. Identify these sequences by a new parameter template_start, and add a copy of install_bp_hardening_cb() that is able to install them. Signed-off-by: James Morse <james.morse@xxxxxxx> Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> --- arch/arm64/include/asm/cpucaps.h | 3 + arch/arm64/include/asm/kvm_mmu.h | 2 - arch/arm64/include/asm/mmu.h | 6 +++ arch/arm64/kernel/bpi.S | 50 +++++++++++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 71 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 128 insertions(+), 4 deletions(-) --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -46,7 +46,8 @@ #define ARM64_MISMATCHED_CACHE_TYPE 26 #define ARM64_SSBS 27 #define ARM64_WORKAROUND_1188873 28 +#define ARM64_SPECTRE_BHB 29 -#define ARM64_NCAPS 29 +#define ARM64_NCAPS 30 #endif /* __ASM_CPUCAPS_H */ --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -358,7 +358,7 @@ static inline void *kvm_get_hyp_vector(v struct bp_hardening_data *data = arm64_get_bp_hardening_data(); void *vect = kvm_ksym_ref(__kvm_hyp_vector); - if (data->fn) { + if (data->template_start) { vect = __bp_harden_hyp_vecs_start + data->hyp_vectors_slot * SZ_2K; --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -46,6 +46,12 @@ typedef void (*bp_hardening_cb_t)(void); struct bp_hardening_data { int hyp_vectors_slot; bp_hardening_cb_t fn; + + /* + * template_start is only used by the BHB mitigation to identify the + * hyp_vectors_slot sequence. + */ + const char *template_start; }; #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR --- a/arch/arm64/kernel/bpi.S +++ b/arch/arm64/kernel/bpi.S @@ -66,3 +66,53 @@ ENTRY(__smccc_workaround_1_smc_start) ldp x0, x1, [sp, #(8 * 2)] add sp, sp, #(8 * 4) ENTRY(__smccc_workaround_1_smc_end) + +ENTRY(__smccc_workaround_3_smc_start) + sub sp, sp, #(8 * 4) + stp x2, x3, [sp, #(8 * 0)] + stp x0, x1, [sp, #(8 * 2)] + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 + smc #0 + ldp x2, x3, [sp, #(8 * 0)] + ldp x0, x1, [sp, #(8 * 2)] + add sp, sp, #(8 * 4) +ENTRY(__smccc_workaround_3_smc_end) + +ENTRY(__spectre_bhb_loop_k8_start) + sub sp, sp, #(8 * 2) + stp x0, x1, [sp, #(8 * 0)] + mov x0, #8 +2: b . + 4 + subs x0, x0, #1 + b.ne 2b + dsb nsh + isb + ldp x0, x1, [sp, #(8 * 0)] + add sp, sp, #(8 * 2) +ENTRY(__spectre_bhb_loop_k8_end) + +ENTRY(__spectre_bhb_loop_k24_start) + sub sp, sp, #(8 * 2) + stp x0, x1, [sp, #(8 * 0)] + mov x0, #24 +2: b . + 4 + subs x0, x0, #1 + b.ne 2b + dsb nsh + isb + ldp x0, x1, [sp, #(8 * 0)] + add sp, sp, #(8 * 2) +ENTRY(__spectre_bhb_loop_k24_end) + +ENTRY(__spectre_bhb_loop_k32_start) + sub sp, sp, #(8 * 2) + stp x0, x1, [sp, #(8 * 0)] + mov x0, #32 +2: b . + 4 + subs x0, x0, #1 + b.ne 2b + dsb nsh + isb + ldp x0, x1, [sp, #(8 * 0)] + add sp, sp, #(8 * 2) +ENTRY(__spectre_bhb_loop_k32_end) --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -85,6 +85,14 @@ DEFINE_PER_CPU_READ_MOSTLY(struct bp_har #ifdef CONFIG_KVM extern char __smccc_workaround_1_smc_start[]; extern char __smccc_workaround_1_smc_end[]; +extern char __smccc_workaround_3_smc_start[]; +extern char __smccc_workaround_3_smc_end[]; +extern char __spectre_bhb_loop_k8_start[]; +extern char __spectre_bhb_loop_k8_end[]; +extern char __spectre_bhb_loop_k24_start[]; +extern char __spectre_bhb_loop_k24_end[]; +extern char __spectre_bhb_loop_k32_start[]; +extern char __spectre_bhb_loop_k32_end[]; static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, const char *hyp_vecs_end) @@ -98,12 +106,14 @@ static void __copy_hyp_vect_bpi(int slot flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K); } +static DEFINE_SPINLOCK(bp_lock); +static int last_slot = -1; + static void install_bp_hardening_cb(bp_hardening_cb_t fn, const char *hyp_vecs_start, const char *hyp_vecs_end) { - static int last_slot = -1; - static DEFINE_SPINLOCK(bp_lock); + int cpu, slot = -1; spin_lock(&bp_lock); @@ -124,6 +134,7 @@ static void install_bp_hardening_cb(bp_h __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); __this_cpu_write(bp_hardening_data.fn, fn); + __this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start); spin_unlock(&bp_lock); } #else @@ -790,3 +801,59 @@ enum mitigation_state arm64_get_spectre_ { return spectre_bhb_state; } + +#ifdef CONFIG_KVM +static const char *kvm_bhb_get_vecs_end(const char *start) +{ + if (start == __smccc_workaround_3_smc_start) + return __smccc_workaround_3_smc_end; + else if (start == __spectre_bhb_loop_k8_start) + return __spectre_bhb_loop_k8_end; + else if (start == __spectre_bhb_loop_k24_start) + return __spectre_bhb_loop_k24_end; + else if (start == __spectre_bhb_loop_k32_start) + return __spectre_bhb_loop_k32_end; + + return NULL; +} + +void kvm_setup_bhb_slot(const char *hyp_vecs_start) +{ + int cpu, slot = -1; + const char *hyp_vecs_end; + + if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available()) + return; + + hyp_vecs_end = kvm_bhb_get_vecs_end(hyp_vecs_start); + if (WARN_ON_ONCE(!hyp_vecs_start || !hyp_vecs_end)) + return; + + spin_lock(&bp_lock); + for_each_possible_cpu(cpu) { + if (per_cpu(bp_hardening_data.template_start, cpu) == hyp_vecs_start) { + slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu); + break; + } + } + + if (slot == -1) { + last_slot++; + BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start) + / SZ_2K) <= last_slot); + slot = last_slot; + __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end); + } + + __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); + __this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start); + spin_unlock(&bp_lock); +} +#else +#define __smccc_workaround_3_smc_start NULL +#define __spectre_bhb_loop_k8_start NULL +#define __spectre_bhb_loop_k24_start NULL +#define __spectre_bhb_loop_k32_start NULL + +void kvm_setup_bhb_slot(const char *hyp_vecs_start) { }; +#endif Patches currently in stable-queue which might be from james.morse@xxxxxxx are queue-4.14/arm64-entry-add-macro-for-reading-symbol-addresses-from-the-trampoline.patch queue-4.14/arm64-use-the-clearbhb-instruction-in-mitigations.patch queue-4.14/arm64-add-percpu-vectors-for-el1.patch queue-4.14/arm64-arch_timer-add-workaround-for-arm-erratum-1188873.patch queue-4.14/arm64-entry-free-up-another-register-on-kpti-s-tramp_exit-path.patch queue-4.14/arm64-entry-don-t-assume-tramp_vectors-is-the-start-of-the-vectors.patch queue-4.14/arm64-entry-make-the-trampoline-cleanup-optional.patch queue-4.14/arm64-add-silicon-errata.txt-entry-for-arm-erratum-1188873.patch queue-4.14/kvm-arm64-add-templates-for-bhb-mitigation-sequences.patch queue-4.14/arm64-entry-add-non-kpti-__bp_harden_el1_vectors-for-mitigations.patch queue-4.14/arm64-add-id_aa64isar2_el1-sys-register.patch queue-4.14/kvm-arm64-allow-smccc_arch_workaround_3-to-be-discovered-and-migrated.patch queue-4.14/arm64-add-neoverse-n2-cortex-a710-cpu-part-definition.patch queue-4.14/arm64-arch_timer-avoid-unused-function-warning.patch queue-4.14/arm64-entry-move-trampoline-macros-out-of-ifdef-d-section.patch queue-4.14/arm64-entry-allow-tramp_alias-to-access-symbols-after-the-4k-boundary.patch queue-4.14/arm64-add-part-number-for-arm-cortex-a77.patch queue-4.14/arm64-entry-move-the-trampoline-data-page-before-the-text-page.patch queue-4.14/arm64-entry.s-add-ventry-overflow-sanity-checks.patch queue-4.14/arm64-add-part-number-for-neoverse-n1.patch queue-4.14/arm64-entry-add-vectors-that-have-the-bhb-mitigation-sequences.patch queue-4.14/arm64-make-arm64_erratum_1188873-depend-on-compat.patch queue-4.14/arm64-mitigate-spectre-style-branch-history-side-channels.patch queue-4.14/arm64-entry-allow-the-trampoline-text-to-occupy-multiple-pages.patch queue-4.14/arm64-proton-pack-report-spectre-bhb-vulnerabilities-as-part-of-spectre-v2.patch queue-4.14/arm64-add-cortex-x2-cpu-part-definition.patch queue-4.14/arm64-entry-make-the-kpti-trampoline-s-kpti-sequence-optional.patch