From: James Morse <james.morse@xxxxxxx> KVM writes the Spectre-v2 mitigation template at the beginning of each vector when a CPU requires a specific sequence to run. Because the template is copied, it can not be modified by the alternatives at runtime. As the KVM template code is intertwined with the bp-hardening callbacks, all templates must have a bp-hardening callback. Add templates for calling ARCH_WORKAROUND_3 and one for each value of K in the brancy-loop. Identify these sequences by a new parameter template_start, and add a copy of install_bp_hardening_cb() that is able to install them. Signed-off-by: James Morse <james.morse@xxxxxxx> Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> --- arch/arm64/include/asm/cpucaps.h | 3 + arch/arm64/include/asm/kvm_mmu.h | 2 - arch/arm64/include/asm/mmu.h | 6 +++ arch/arm64/kernel/bpi.S | 50 +++++++++++++++++++++++++++ arch/arm64/kernel/cpu_errata.c | 71 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 128 insertions(+), 4 deletions(-) --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -39,7 +39,8 @@ #define ARM64_SSBD 18 #define ARM64_MISMATCHED_CACHE_TYPE 19 #define ARM64_WORKAROUND_1188873 20 +#define ARM64_SPECTRE_BHB 21 -#define ARM64_NCAPS 21 +#define ARM64_NCAPS 22 #endif /* __ASM_CPUCAPS_H */ --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -362,7 +362,7 @@ static inline void *kvm_get_hyp_vector(v struct bp_hardening_data *data = arm64_get_bp_hardening_data(); void *vect = kvm_ksym_ref(__kvm_hyp_vector); - if (data->fn) { + if (data->template_start) { vect = __bp_harden_hyp_vecs_start + data->hyp_vectors_slot * SZ_2K; --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -45,6 +45,12 @@ typedef void (*bp_hardening_cb_t)(void); struct bp_hardening_data { int hyp_vectors_slot; bp_hardening_cb_t fn; + + /* + * template_start is only used by the BHB mitigation to identify the + * hyp_vectors_slot sequence. + */ + const char *template_start; }; #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR --- a/arch/arm64/kernel/bpi.S +++ b/arch/arm64/kernel/bpi.S @@ -73,3 +73,53 @@ ENTRY(__smccc_workaround_1_smc_end) ENTRY(__smccc_workaround_1_hvc_start) smccc_workaround_1 hvc ENTRY(__smccc_workaround_1_hvc_end) + +ENTRY(__smccc_workaround_3_smc_start) + sub sp, sp, #(8 * 4) + stp x2, x3, [sp, #(8 * 0)] + stp x0, x1, [sp, #(8 * 2)] + mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 + smc #0 + ldp x2, x3, [sp, #(8 * 0)] + ldp x0, x1, [sp, #(8 * 2)] + add sp, sp, #(8 * 4) +ENTRY(__smccc_workaround_3_smc_end) + +ENTRY(__spectre_bhb_loop_k8_start) + sub sp, sp, #(8 * 2) + stp x0, x1, [sp, #(8 * 0)] + mov x0, #8 +2: b . + 4 + subs x0, x0, #1 + b.ne 2b + dsb nsh + isb + ldp x0, x1, [sp, #(8 * 0)] + add sp, sp, #(8 * 2) +ENTRY(__spectre_bhb_loop_k8_end) + +ENTRY(__spectre_bhb_loop_k24_start) + sub sp, sp, #(8 * 2) + stp x0, x1, [sp, #(8 * 0)] + mov x0, #24 +2: b . + 4 + subs x0, x0, #1 + b.ne 2b + dsb nsh + isb + ldp x0, x1, [sp, #(8 * 0)] + add sp, sp, #(8 * 2) +ENTRY(__spectre_bhb_loop_k24_end) + +ENTRY(__spectre_bhb_loop_k32_start) + sub sp, sp, #(8 * 2) + stp x0, x1, [sp, #(8 * 0)] + mov x0, #32 +2: b . + 4 + subs x0, x0, #1 + b.ne 2b + dsb nsh + isb + ldp x0, x1, [sp, #(8 * 0)] + add sp, sp, #(8 * 2) +ENTRY(__spectre_bhb_loop_k32_end) --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -74,6 +74,14 @@ extern char __smccc_workaround_1_smc_sta extern char __smccc_workaround_1_smc_end[]; extern char __smccc_workaround_1_hvc_start[]; extern char __smccc_workaround_1_hvc_end[]; +extern char __smccc_workaround_3_smc_start[]; +extern char __smccc_workaround_3_smc_end[]; +extern char __spectre_bhb_loop_k8_start[]; +extern char __spectre_bhb_loop_k8_end[]; +extern char __spectre_bhb_loop_k24_start[]; +extern char __spectre_bhb_loop_k24_end[]; +extern char __spectre_bhb_loop_k32_start[]; +extern char __spectre_bhb_loop_k32_end[]; static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, const char *hyp_vecs_end) @@ -87,12 +95,14 @@ static void __copy_hyp_vect_bpi(int slot flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K); } +static DEFINE_SPINLOCK(bp_lock); +static int last_slot = -1; + static void __install_bp_hardening_cb(bp_hardening_cb_t fn, const char *hyp_vecs_start, const char *hyp_vecs_end) { - static int last_slot = -1; - static DEFINE_SPINLOCK(bp_lock); + int cpu, slot = -1; spin_lock(&bp_lock); @@ -113,6 +123,7 @@ static void __install_bp_hardening_cb(bp __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); __this_cpu_write(bp_hardening_data.fn, fn); + __this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start); spin_unlock(&bp_lock); } #else @@ -544,3 +555,59 @@ const struct arm64_cpu_capabilities arm6 { } }; + +#ifdef CONFIG_KVM +static const char *kvm_bhb_get_vecs_end(const char *start) +{ + if (start == __smccc_workaround_3_smc_start) + return __smccc_workaround_3_smc_end; + else if (start == __spectre_bhb_loop_k8_start) + return __spectre_bhb_loop_k8_end; + else if (start == __spectre_bhb_loop_k24_start) + return __spectre_bhb_loop_k24_end; + else if (start == __spectre_bhb_loop_k32_start) + return __spectre_bhb_loop_k32_end; + + return NULL; +} + +void kvm_setup_bhb_slot(const char *hyp_vecs_start) +{ + int cpu, slot = -1; + const char *hyp_vecs_end; + + if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available()) + return; + + hyp_vecs_end = kvm_bhb_get_vecs_end(hyp_vecs_start); + if (WARN_ON_ONCE(!hyp_vecs_start || !hyp_vecs_end)) + return; + + spin_lock(&bp_lock); + for_each_possible_cpu(cpu) { + if (per_cpu(bp_hardening_data.template_start, cpu) == hyp_vecs_start) { + slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu); + break; + } + } + + if (slot == -1) { + last_slot++; + BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start) + / SZ_2K) <= last_slot); + slot = last_slot; + __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end); + } + + __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot); + __this_cpu_write(bp_hardening_data.template_start, hyp_vecs_start); + spin_unlock(&bp_lock); +} +#else +#define __smccc_workaround_3_smc_start NULL +#define __spectre_bhb_loop_k8_start NULL +#define __spectre_bhb_loop_k24_start NULL +#define __spectre_bhb_loop_k32_start NULL + +void kvm_setup_bhb_slot(const char *hyp_vecs_start) { }; +#endif Patches currently in stable-queue which might be from james.morse@xxxxxxx are queue-4.9/clocksource-drivers-arm_arch_timer-remove-fsl-a008585-parameter.patch queue-4.9/arm64-capabilities-clean-up-midr-range-helpers.patch queue-4.9/arm64-entry-add-macro-for-reading-symbol-addresses-from-the-trampoline.patch queue-4.9/arm64-use-the-clearbhb-instruction-in-mitigations.patch queue-4.9/arm64-add-percpu-vectors-for-el1.patch queue-4.9/arm64-arch_timer-add-workaround-for-arm-erratum-1188873.patch queue-4.9/arm64-capabilities-add-support-for-checks-based-on-a-list-of-midrs.patch queue-4.9/arm64-arch_timer-add-infrastructure-for-multiple-erratum-detection-methods.patch queue-4.9/arm64-entry-free-up-another-register-on-kpti-s-tramp_exit-path.patch queue-4.9/arm64-entry-don-t-assume-tramp_vectors-is-the-start-of-the-vectors.patch queue-4.9/arm64-entry-make-the-trampoline-cleanup-optional.patch queue-4.9/arm64-add-silicon-errata.txt-entry-for-arm-erratum-1188873.patch queue-4.9/clocksource-drivers-arm_arch_timer-introduce-generic-errata-handling-infrastructure.patch queue-4.9/kvm-arm64-add-templates-for-bhb-mitigation-sequences.patch queue-4.9/arm64-entry-add-non-kpti-__bp_harden_el1_vectors-for-mitigations.patch queue-4.9/arm64-add-id_aa64isar2_el1-sys-register.patch queue-4.9/kvm-arm64-allow-smccc_arch_workaround_3-to-be-discovered-and-migrated.patch queue-4.9/arm64-capabilities-update-prototype-for-enable-call-back.patch queue-4.9/arm64-add-neoverse-n2-cortex-a710-cpu-part-definition.patch queue-4.9/arm64-add-midr-encoding-for-arm-cortex-a55-and-cortex-a35.patch queue-4.9/arm64-capabilities-move-errata-processing-code.patch queue-4.9/arm64-arch_timer-add-erratum-handler-for-cpu-specific-capability.patch queue-4.9/arm64-errata-provide-macro-for-major-and-minor-cpu-revisions.patch queue-4.9/arm64-arch_timer-avoid-unused-function-warning.patch queue-4.9/arm64-capabilities-move-errata-work-around-check-on-boot-cpu.patch queue-4.9/arm64-add-helpers-for-checking-cpu-midr-against-a-range.patch queue-4.9/arm64-entry-move-trampoline-macros-out-of-ifdef-d-section.patch queue-4.9/arm64-entry-allow-tramp_alias-to-access-symbols-after-the-4k-boundary.patch queue-4.9/arm64-add-part-number-for-arm-cortex-a77.patch queue-4.9/arm64-capabilities-add-flags-to-handle-the-conflicts-on-late-cpu.patch queue-4.9/arm64-entry-move-the-trampoline-data-page-before-the-text-page.patch queue-4.9/arm64-entry.s-add-ventry-overflow-sanity-checks.patch queue-4.9/arm64-add-part-number-for-neoverse-n1.patch queue-4.9/arm64-entry-add-vectors-that-have-the-bhb-mitigation-sequences.patch queue-4.9/arm64-make-arm64_erratum_1188873-depend-on-compat.patch queue-4.9/arm64-mitigate-spectre-style-branch-history-side-channels.patch queue-4.9/arm64-move-arm64_update_smccc_conduit-out-of-ssbd-ifdef.patch queue-4.9/arm64-entry-allow-the-trampoline-text-to-occupy-multiple-pages.patch queue-4.9/arm64-add-cortex-x2-cpu-part-definition.patch queue-4.9/arm64-add-helper-to-decode-register-from-instruction.patch queue-4.9/arm64-capabilities-prepare-for-fine-grained-capabilities.patch queue-4.9/arm64-remove-useless-uao-ipi-and-describe-how-this-gets-enabled.patch queue-4.9/arm64-entry-make-the-kpti-trampoline-s-kpti-sequence-optional.patch