On Mon, Jan 22, 2024 at 03:52:40PM -0800, isaku.yamahata@xxxxxxxxx wrote:
> From: Sean Christopherson <sean.j.christopherson@xxxxxxxxx>
>
> KVM accesses Virtual Machine Control Structure (VMCS) with VMX instructions
> to operate on VM. TDX doesn't allow VMM to operate VMCS directly.
> Instead, TDX has its own data structures, and TDX SEAMCALL APIs for VMM to
> indirectly operate those data structures. This means we must have a TDX
> version of kvm_x86_ops.
>
> The existing global struct kvm_x86_ops already defines an interface which
> fits with TDX. But kvm_x86_ops is system-wide, not per-VM structure. To
> allow VMX to coexist with TDs, the kvm_x86_ops callbacks will have wrappers
> "if (tdx) tdx_op() else vmx_op()" to switch VMX or TDX at run time.
>
> To split the runtime switch, the VMX implementation, and the TDX
> implementation, add main.c, and move out the vmx_x86_ops hooks in
> preparation for adding TDX, which can coexist with VMX, i.e. KVM can run
> both VMs and TDs. Use 'vt' for the naming scheme as a nod to VT-x and as a
> concatenation of VmxTdx.
>
> The current code looks as follows.
> In vmx.c
> static vmx_op() { ... }
> static struct kvm_x86_ops vmx_x86_ops = {
> .op = vmx_op,
> initialization code
>
> The eventually converted code will look like
> In vmx.c, keep the VMX operations.
> vmx_op() { ... }
> VMX initialization
> In tdx.c, define the TDX operations.
> tdx_op() { ... }
> TDX initialization
> In x86_ops.h, declare the VMX and TDX operations.
> vmx_op();
> tdx_op();
> In main.c, define common wrappers for VMX and TDX.
> static vt_ops() { if (tdx) tdx_ops() else vmx_ops() }
> static struct kvm_x86_ops vt_x86_ops = {
> .op = vt_op,
> initialization to call VMX and TDX initialization
>
> Opportunistically, fix the name inconsistency from vmx_create_vcpu() and
> vmx_free_vcpu() to vmx_vcpu_create() and vmx_vcpu_free().
>
> Co-developed-by: Xiaoyao Li <xiaoyao.li@xxxxxxxxx>
> Signed-off-by: Xiaoyao Li <xiaoyao.li@xxxxxxxxx>
> Signed-off-by: Sean Christopherson <sean.j.christopherson@xxxxxxxxx>
> Signed-off-by: Isaku Yamahata <isaku.yamahata@xxxxxxxxx>
> Reviewed-by: Paolo Bonzini <pbonzini@xxxxxxxxxx>
> Reviewed-by: Binbin Wu <binbin.wu@xxxxxxxxxxxxxxx>
Reviewed-by: Yuan Yao <yuan.yao@xxxxxxxxx>
>
> ---
> v18:
> - Add Reviewed-by: Binbin Wu
> - fix indent alignments pointed by Binbin Wu
> ---
> arch/x86/kvm/Makefile | 2 +-
> arch/x86/kvm/vmx/main.c | 169 +++++++++++++++++
> arch/x86/kvm/vmx/vmx.c | 378 ++++++++++---------------------------
> arch/x86/kvm/vmx/x86_ops.h | 125 ++++++++++++
> 4 files changed, 398 insertions(+), 276 deletions(-)
> create mode 100644 arch/x86/kvm/vmx/main.c
> create mode 100644 arch/x86/kvm/vmx/x86_ops.h
>
> diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
> index 475b5fa917a6..274df24b647f 100644
> --- a/arch/x86/kvm/Makefile
> +++ b/arch/x86/kvm/Makefile
> @@ -20,7 +20,7 @@ kvm-$(CONFIG_KVM_XEN) += xen.o
> kvm-$(CONFIG_KVM_SMM) += smm.o
>
> kvm-intel-y += vmx/vmx.o vmx/vmenter.o vmx/pmu_intel.o vmx/vmcs12.o \
> - vmx/nested.o vmx/posted_intr.o
> + vmx/nested.o vmx/posted_intr.o vmx/main.o
>
> kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o
> kvm-intel-$(CONFIG_KVM_HYPERV) += vmx/hyperv.o vmx/hyperv_evmcs.o
> diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
> new file mode 100644
> index 000000000000..eeb7a43b271d
> --- /dev/null
> +++ b/arch/x86/kvm/vmx/main.c
> @@ -0,0 +1,169 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#include <linux/moduleparam.h>
> +
> +#include "x86_ops.h"
> +#include "vmx.h"
> +#include "nested.h"
> +#include "pmu.h"
> +
> +#define VMX_REQUIRED_APICV_INHIBITS \
> + (BIT(APICV_INHIBIT_REASON_DISABLE)| \
> + BIT(APICV_INHIBIT_REASON_ABSENT) | \
> + BIT(APICV_INHIBIT_REASON_HYPERV) | \
> + BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | \
> + BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \
> + BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | \
> + BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED))
> +
> +struct kvm_x86_ops vt_x86_ops __initdata = {
> + .name = KBUILD_MODNAME,
> +
> + .check_processor_compatibility = vmx_check_processor_compat,
> +
> + .hardware_unsetup = vmx_hardware_unsetup,
> +
> + .hardware_enable = vmx_hardware_enable,
> + .hardware_disable = vmx_hardware_disable,
> + .has_emulated_msr = vmx_has_emulated_msr,
> +
> + .is_vm_type_supported = vmx_is_vm_type_supported,
> + .vm_size = sizeof(struct kvm_vmx),
> + .vm_init = vmx_vm_init,
> + .vm_destroy = vmx_vm_destroy,
> +
> + .vcpu_precreate = vmx_vcpu_precreate,
> + .vcpu_create = vmx_vcpu_create,
> + .vcpu_free = vmx_vcpu_free,
> + .vcpu_reset = vmx_vcpu_reset,
> +
> + .prepare_switch_to_guest = vmx_prepare_switch_to_guest,
> + .vcpu_load = vmx_vcpu_load,
> + .vcpu_put = vmx_vcpu_put,
> +
> + .update_exception_bitmap = vmx_update_exception_bitmap,
> + .get_msr_feature = vmx_get_msr_feature,
> + .get_msr = vmx_get_msr,
> + .set_msr = vmx_set_msr,
> + .get_segment_base = vmx_get_segment_base,
> + .get_segment = vmx_get_segment,
> + .set_segment = vmx_set_segment,
> + .get_cpl = vmx_get_cpl,
> + .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
> + .is_valid_cr0 = vmx_is_valid_cr0,
> + .set_cr0 = vmx_set_cr0,
> + .is_valid_cr4 = vmx_is_valid_cr4,
> + .set_cr4 = vmx_set_cr4,
> + .set_efer = vmx_set_efer,
> + .get_idt = vmx_get_idt,
> + .set_idt = vmx_set_idt,
> + .get_gdt = vmx_get_gdt,
> + .set_gdt = vmx_set_gdt,
> + .set_dr7 = vmx_set_dr7,
> + .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
> + .cache_reg = vmx_cache_reg,
> + .get_rflags = vmx_get_rflags,
> + .set_rflags = vmx_set_rflags,
> + .get_if_flag = vmx_get_if_flag,
> +
> + .flush_tlb_all = vmx_flush_tlb_all,
> + .flush_tlb_current = vmx_flush_tlb_current,
> + .flush_tlb_gva = vmx_flush_tlb_gva,
> + .flush_tlb_guest = vmx_flush_tlb_guest,
> +
> + .vcpu_pre_run = vmx_vcpu_pre_run,
> + .vcpu_run = vmx_vcpu_run,
> + .handle_exit = vmx_handle_exit,
> + .skip_emulated_instruction = vmx_skip_emulated_instruction,
> + .update_emulated_instruction = vmx_update_emulated_instruction,
> + .set_interrupt_shadow = vmx_set_interrupt_shadow,
> + .get_interrupt_shadow = vmx_get_interrupt_shadow,
> + .patch_hypercall = vmx_patch_hypercall,
> + .inject_irq = vmx_inject_irq,
> + .inject_nmi = vmx_inject_nmi,
> + .inject_exception = vmx_inject_exception,
> + .cancel_injection = vmx_cancel_injection,
> + .interrupt_allowed = vmx_interrupt_allowed,
> + .nmi_allowed = vmx_nmi_allowed,
> + .get_nmi_mask = vmx_get_nmi_mask,
> + .set_nmi_mask = vmx_set_nmi_mask,
> + .enable_nmi_window = vmx_enable_nmi_window,
> + .enable_irq_window = vmx_enable_irq_window,
> + .update_cr8_intercept = vmx_update_cr8_intercept,
> + .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
> + .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
> + .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
> + .load_eoi_exitmap = vmx_load_eoi_exitmap,
> + .apicv_pre_state_restore = vmx_apicv_pre_state_restore,
> + .required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS,
> + .hwapic_irr_update = vmx_hwapic_irr_update,
> + .hwapic_isr_update = vmx_hwapic_isr_update,
> + .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
> + .sync_pir_to_irr = vmx_sync_pir_to_irr,
> + .deliver_interrupt = vmx_deliver_interrupt,
> + .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt,
> +
> + .set_tss_addr = vmx_set_tss_addr,
> + .set_identity_map_addr = vmx_set_identity_map_addr,
> + .get_mt_mask = vmx_get_mt_mask,
> +
> + .get_exit_info = vmx_get_exit_info,
> +
> + .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid,
> +
> + .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
> +
> + .get_l2_tsc_offset = vmx_get_l2_tsc_offset,
> + .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier,
> + .write_tsc_offset = vmx_write_tsc_offset,
> + .write_tsc_multiplier = vmx_write_tsc_multiplier,
> +
> + .load_mmu_pgd = vmx_load_mmu_pgd,
> +
> + .check_intercept = vmx_check_intercept,
> + .handle_exit_irqoff = vmx_handle_exit_irqoff,
> +
> + .request_immediate_exit = vmx_request_immediate_exit,
> +
> + .sched_in = vmx_sched_in,
> +
> + .cpu_dirty_log_size = PML_ENTITY_NUM,
> + .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging,
> +
> + .nested_ops = &vmx_nested_ops,
> +
> + .pi_update_irte = vmx_pi_update_irte,
> + .pi_start_assignment = vmx_pi_start_assignment,
> +
> +#ifdef CONFIG_X86_64
> + .set_hv_timer = vmx_set_hv_timer,
> + .cancel_hv_timer = vmx_cancel_hv_timer,
> +#endif
> +
> + .setup_mce = vmx_setup_mce,
> +
> +#ifdef CONFIG_KVM_SMM
> + .smi_allowed = vmx_smi_allowed,
> + .enter_smm = vmx_enter_smm,
> + .leave_smm = vmx_leave_smm,
> + .enable_smi_window = vmx_enable_smi_window,
> +#endif
> +
> + .check_emulate_instruction = vmx_check_emulate_instruction,
> + .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
> + .migrate_timers = vmx_migrate_timers,
> +
> + .msr_filter_changed = vmx_msr_filter_changed,
> + .complete_emulated_msr = kvm_complete_insn_gp,
> +
> + .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
> +
> + .get_untagged_addr = vmx_get_untagged_addr,
> +};
> +
> +struct kvm_x86_init_ops vt_init_ops __initdata = {
> + .hardware_setup = vmx_hardware_setup,
> + .handle_intel_pt_intr = NULL,
> +
> + .runtime_ops = &vt_x86_ops,
> + .pmu_ops = &intel_pmu_ops,
> +};
> diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
> index 8fad7bba6d5f..55597b3bdc55 100644
> --- a/arch/x86/kvm/vmx/vmx.c
> +++ b/arch/x86/kvm/vmx/vmx.c
> @@ -65,6 +65,7 @@
> #include "vmcs12.h"
> #include "vmx.h"
> #include "x86.h"
> +#include "x86_ops.h"
> #include "smm.h"
> #include "vmx_onhyperv.h"
>
> @@ -516,8 +517,6 @@ static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
> static unsigned long host_idt_base;
>
> #if IS_ENABLED(CONFIG_HYPERV)
> -static struct kvm_x86_ops vmx_x86_ops __initdata;
> -
> static bool __read_mostly enlightened_vmcs = true;
> module_param(enlightened_vmcs, bool, 0444);
>
> @@ -567,9 +566,8 @@ static __init void hv_init_evmcs(void)
> }
>
> if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH)
> - vmx_x86_ops.enable_l2_tlb_flush
> + vt_x86_ops.enable_l2_tlb_flush
> = hv_enable_l2_tlb_flush;
> -
> } else {
> enlightened_vmcs = false;
> }
> @@ -1474,7 +1472,7 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
> * Switches to specified vcpu, until a matching vcpu_put(), but assumes
> * vcpu mutex is already taken.
> */
> -static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
> +void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -1485,7 +1483,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
> vmx->host_debugctlmsr = get_debugctlmsr();
> }
>
> -static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
> +void vmx_vcpu_put(struct kvm_vcpu *vcpu)
> {
> vmx_vcpu_pi_put(vcpu);
>
> @@ -1544,7 +1542,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
> vmx->emulation_required = vmx_emulation_required(vcpu);
> }
>
> -static bool vmx_get_if_flag(struct kvm_vcpu *vcpu)
> +bool vmx_get_if_flag(struct kvm_vcpu *vcpu)
> {
> return vmx_get_rflags(vcpu) & X86_EFLAGS_IF;
> }
> @@ -1650,8 +1648,8 @@ static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data)
> return 0;
> }
>
> -static int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
> - void *insn, int insn_len)
> +int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
> + void *insn, int insn_len)
> {
> /*
> * Emulation of instructions in SGX enclaves is impossible as RIP does
> @@ -1735,7 +1733,7 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
> * Recognizes a pending MTF VM-exit and records the nested state for later
> * delivery.
> */
> -static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
> +void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
> {
> struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> @@ -1766,7 +1764,7 @@ static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu)
> }
> }
>
> -static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
> +int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu)
> {
> vmx_update_emulated_instruction(vcpu);
> return skip_emulated_instruction(vcpu);
> @@ -1785,7 +1783,7 @@ static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
> vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
> }
>
> -static void vmx_inject_exception(struct kvm_vcpu *vcpu)
> +void vmx_inject_exception(struct kvm_vcpu *vcpu)
> {
> struct kvm_queued_exception *ex = &vcpu->arch.exception;
> u32 intr_info = ex->vector | INTR_INFO_VALID_MASK;
> @@ -1906,12 +1904,12 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
> return kvm_caps.default_tsc_scaling_ratio;
> }
>
> -static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu)
> +void vmx_write_tsc_offset(struct kvm_vcpu *vcpu)
> {
> vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
> }
>
> -static void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu)
> +void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu)
> {
> vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio);
> }
> @@ -1954,7 +1952,7 @@ static inline bool is_vmx_feature_control_msr_valid(struct vcpu_vmx *vmx,
> return !(msr->data & ~valid_bits);
> }
>
> -static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
> +int vmx_get_msr_feature(struct kvm_msr_entry *msr)
> {
> switch (msr->index) {
> case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
> @@ -1971,7 +1969,7 @@ static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
> * Returns 0 on success, non-0 otherwise.
> * Assumes vcpu_load() was already called.
> */
> -static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> +int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> struct vmx_uret_msr *msr;
> @@ -2152,7 +2150,7 @@ static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated
> * Returns 0 on success, non-0 otherwise.
> * Assumes vcpu_load() was already called.
> */
> -static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> +int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> struct vmx_uret_msr *msr;
> @@ -2455,7 +2453,7 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
> return ret;
> }
>
> -static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
> +void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
> {
> unsigned long guest_owned_bits;
>
> @@ -2756,7 +2754,7 @@ static bool kvm_is_vmx_supported(void)
> return supported;
> }
>
> -static int vmx_check_processor_compat(void)
> +int vmx_check_processor_compat(void)
> {
> int cpu = raw_smp_processor_id();
> struct vmcs_config vmcs_conf;
> @@ -2798,7 +2796,7 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer)
> return -EFAULT;
> }
>
> -static int vmx_hardware_enable(void)
> +int vmx_hardware_enable(void)
> {
> int cpu = raw_smp_processor_id();
> u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
> @@ -2838,7 +2836,7 @@ static void vmclear_local_loaded_vmcss(void)
> __loaded_vmcs_clear(v);
> }
>
> -static void vmx_hardware_disable(void)
> +void vmx_hardware_disable(void)
> {
> vmclear_local_loaded_vmcss();
>
> @@ -3152,7 +3150,7 @@ static void exit_lmode(struct kvm_vcpu *vcpu)
>
> #endif
>
> -static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
> +void vmx_flush_tlb_all(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -3182,7 +3180,7 @@ static inline int vmx_get_current_vpid(struct kvm_vcpu *vcpu)
> return to_vmx(vcpu)->vpid;
> }
>
> -static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
> +void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
> {
> struct kvm_mmu *mmu = vcpu->arch.mmu;
> u64 root_hpa = mmu->root.hpa;
> @@ -3198,7 +3196,7 @@ static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu)
> vpid_sync_context(vmx_get_current_vpid(vcpu));
> }
>
> -static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
> +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
> {
> /*
> * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in
> @@ -3207,7 +3205,7 @@ static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
> vpid_sync_vcpu_addr(vmx_get_current_vpid(vcpu), addr);
> }
>
> -static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
> +void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu)
> {
> /*
> * vpid_sync_context() is a nop if vpid==0, e.g. if enable_vpid==0 or a
> @@ -3252,7 +3250,7 @@ void ept_save_pdptrs(struct kvm_vcpu *vcpu)
> #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \
> CPU_BASED_CR3_STORE_EXITING)
>
> -static bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
> +bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
> {
> if (is_guest_mode(vcpu))
> return nested_guest_cr0_valid(vcpu, cr0);
> @@ -3373,8 +3371,7 @@ u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level)
> return eptp;
> }
>
> -static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
> - int root_level)
> +void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level)
> {
> struct kvm *kvm = vcpu->kvm;
> bool update_guest_cr3 = true;
> @@ -3403,8 +3400,7 @@ static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
> vmcs_writel(GUEST_CR3, guest_cr3);
> }
>
> -
> -static bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
> +bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
> {
> /*
> * We operate under the default treatment of SMM, so VMX cannot be
> @@ -3520,7 +3516,7 @@ void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
> var->g = (ar >> 15) & 1;
> }
>
> -static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
> +u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
> {
> struct kvm_segment s;
>
> @@ -3597,14 +3593,14 @@ void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
> vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
> }
>
> -static void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
> +void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg)
> {
> __vmx_set_segment(vcpu, var, seg);
>
> to_vmx(vcpu)->emulation_required = vmx_emulation_required(vcpu);
> }
>
> -static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
> +void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
> {
> u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
>
> @@ -3612,25 +3608,25 @@ static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
> *l = (ar >> 13) & 1;
> }
>
> -static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> +void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> {
> dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
> dt->address = vmcs_readl(GUEST_IDTR_BASE);
> }
>
> -static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> +void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> {
> vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
> vmcs_writel(GUEST_IDTR_BASE, dt->address);
> }
>
> -static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> +void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> {
> dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
> dt->address = vmcs_readl(GUEST_GDTR_BASE);
> }
>
> -static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> +void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
> {
> vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
> vmcs_writel(GUEST_GDTR_BASE, dt->address);
> @@ -4102,7 +4098,7 @@ void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu)
> }
> }
>
> -static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
> +bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> void *vapic_page;
> @@ -4122,7 +4118,7 @@ static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
> return ((rvi & 0xf0) > (vppr & 0xf0));
> }
>
> -static void vmx_msr_filter_changed(struct kvm_vcpu *vcpu)
> +void vmx_msr_filter_changed(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> u32 i;
> @@ -4263,8 +4259,8 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
> return 0;
> }
>
> -static void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
> - int trig_mode, int vector)
> +void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
> + int trig_mode, int vector)
> {
> struct kvm_vcpu *vcpu = apic->vcpu;
>
> @@ -4426,7 +4422,7 @@ static u32 vmx_vmexit_ctrl(void)
> ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER);
> }
>
> -static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
> +void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -4690,7 +4686,7 @@ static int vmx_alloc_ipiv_pid_table(struct kvm *kvm)
> return 0;
> }
>
> -static int vmx_vcpu_precreate(struct kvm *kvm)
> +int vmx_vcpu_precreate(struct kvm *kvm)
> {
> return vmx_alloc_ipiv_pid_table(kvm);
> }
> @@ -4845,7 +4841,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu)
> vmx->pi_desc.sn = 1;
> }
>
> -static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
> +void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -4904,12 +4900,12 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
> vmx_update_fb_clear_dis(vcpu, vmx);
> }
>
> -static void vmx_enable_irq_window(struct kvm_vcpu *vcpu)
> +void vmx_enable_irq_window(struct kvm_vcpu *vcpu)
> {
> exec_controls_setbit(to_vmx(vcpu), CPU_BASED_INTR_WINDOW_EXITING);
> }
>
> -static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
> +void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
> {
> if (!enable_vnmi ||
> vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
> @@ -4920,7 +4916,7 @@ static void vmx_enable_nmi_window(struct kvm_vcpu *vcpu)
> exec_controls_setbit(to_vmx(vcpu), CPU_BASED_NMI_WINDOW_EXITING);
> }
>
> -static void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
> +void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> uint32_t intr;
> @@ -4948,7 +4944,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
> vmx_clear_hlt(vcpu);
> }
>
> -static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
> +void vmx_inject_nmi(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -5026,7 +5022,7 @@ bool vmx_nmi_blocked(struct kvm_vcpu *vcpu)
> GUEST_INTR_STATE_NMI));
> }
>
> -static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> +int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> {
> if (to_vmx(vcpu)->nested.nested_run_pending)
> return -EBUSY;
> @@ -5048,7 +5044,7 @@ bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu)
> (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
> }
>
> -static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> +int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> {
> if (to_vmx(vcpu)->nested.nested_run_pending)
> return -EBUSY;
> @@ -5063,7 +5059,7 @@ static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> return !vmx_interrupt_blocked(vcpu);
> }
>
> -static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
> +int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
> {
> void __user *ret;
>
> @@ -5083,7 +5079,7 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
> return init_rmode_tss(kvm, ret);
> }
>
> -static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
> +int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
> {
> to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
> return 0;
> @@ -5369,8 +5365,7 @@ static int handle_io(struct kvm_vcpu *vcpu)
> return kvm_fast_pio(vcpu, size, port, in);
> }
>
> -static void
> -vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
> +void vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
> {
> /*
> * Patch in the VMCALL instruction:
> @@ -5579,7 +5574,7 @@ static int handle_dr(struct kvm_vcpu *vcpu)
> return kvm_complete_insn_gp(vcpu, err);
> }
>
> -static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
> +void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
> {
> get_debugreg(vcpu->arch.db[0], 0);
> get_debugreg(vcpu->arch.db[1], 1);
> @@ -5598,7 +5593,7 @@ static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
> set_debugreg(DR6_RESERVED, 6);
> }
>
> -static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
> +void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
> {
> vmcs_writel(GUEST_DR7, val);
> }
> @@ -5869,7 +5864,7 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
> return 1;
> }
>
> -static int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu)
> +int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu)
> {
> if (vmx_emulation_required_with_pending_exception(vcpu)) {
> kvm_prepare_emulation_failure_exit(vcpu);
> @@ -6133,9 +6128,8 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
> static const int kvm_vmx_max_exit_handlers =
> ARRAY_SIZE(kvm_vmx_exit_handlers);
>
> -static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
> - u64 *info1, u64 *info2,
> - u32 *intr_info, u32 *error_code)
> +void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
> + u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -6578,7 +6572,7 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
> return 0;
> }
>
> -static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
> +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
> {
> int ret = __vmx_handle_exit(vcpu, exit_fastpath);
>
> @@ -6666,7 +6660,7 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
> : "eax", "ebx", "ecx", "edx");
> }
>
> -static void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
> +void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
> {
> struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
> int tpr_threshold;
> @@ -6736,7 +6730,7 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
> vmx_update_msr_bitmap_x2apic(vcpu);
> }
>
> -static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
> +void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
> {
> const gfn_t gfn = APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT;
> struct kvm *kvm = vcpu->kvm;
> @@ -6805,7 +6799,7 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
> kvm_release_pfn_clean(pfn);
> }
>
> -static void vmx_hwapic_isr_update(int max_isr)
> +void vmx_hwapic_isr_update(int max_isr)
> {
> u16 status;
> u8 old;
> @@ -6839,7 +6833,7 @@ static void vmx_set_rvi(int vector)
> }
> }
>
> -static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
> +void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
> {
> /*
> * When running L2, updating RVI is only relevant when
> @@ -6853,7 +6847,7 @@ static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
> vmx_set_rvi(max_irr);
> }
>
> -static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
> +int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> int max_irr;
> @@ -6899,7 +6893,7 @@ static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
> return max_irr;
> }
>
> -static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
> +void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
> {
> if (!kvm_vcpu_apicv_active(vcpu))
> return;
> @@ -6910,7 +6904,7 @@ static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
> vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
> }
>
> -static void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu)
> +void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -6973,7 +6967,7 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu)
> vcpu->arch.at_instruction_boundary = true;
> }
>
> -static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
> +void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -6990,7 +6984,7 @@ static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu)
> * The kvm parameter can be NULL (module initialization, or invocation before
> * VM creation). Be sure to check the kvm parameter before using it.
> */
> -static bool vmx_has_emulated_msr(struct kvm *kvm, u32 index)
> +bool vmx_has_emulated_msr(struct kvm *kvm, u32 index)
> {
> switch (index) {
> case MSR_IA32_SMBASE:
> @@ -7113,7 +7107,7 @@ static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
> IDT_VECTORING_ERROR_CODE);
> }
>
> -static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
> +void vmx_cancel_injection(struct kvm_vcpu *vcpu)
> {
> __vmx_complete_interrupts(vcpu,
> vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
> @@ -7268,7 +7262,7 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
> guest_state_exit_irqoff();
> }
>
> -static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
> +fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> unsigned long cr3, cr4;
> @@ -7424,7 +7418,7 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu)
> return vmx_exit_handlers_fastpath(vcpu);
> }
>
> -static void vmx_vcpu_free(struct kvm_vcpu *vcpu)
> +void vmx_vcpu_free(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -7435,7 +7429,7 @@ static void vmx_vcpu_free(struct kvm_vcpu *vcpu)
> free_loaded_vmcs(vmx->loaded_vmcs);
> }
>
> -static int vmx_vcpu_create(struct kvm_vcpu *vcpu)
> +int vmx_vcpu_create(struct kvm_vcpu *vcpu)
> {
> struct vmx_uret_msr *tsx_ctrl;
> struct vcpu_vmx *vmx;
> @@ -7541,7 +7535,7 @@ static int vmx_vcpu_create(struct kvm_vcpu *vcpu)
> return err;
> }
>
> -static bool vmx_is_vm_type_supported(unsigned long type)
> +bool vmx_is_vm_type_supported(unsigned long type)
> {
> /* TODO: Check if TDX is supported. */
> return __kvm_is_vm_type_supported(type);
> @@ -7550,7 +7544,7 @@ static bool vmx_is_vm_type_supported(unsigned long type)
> #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
> #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
>
> -static int vmx_vm_init(struct kvm *kvm)
> +int vmx_vm_init(struct kvm *kvm)
> {
> if (!ple_gap)
> kvm->arch.pause_in_guest = true;
> @@ -7581,7 +7575,7 @@ static int vmx_vm_init(struct kvm *kvm)
> return 0;
> }
>
> -static u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
> +u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
> {
> /* We wanted to honor guest CD/MTRR/PAT, but doing so could result in
> * memory aliases with conflicting memory types and sometimes MCEs.
> @@ -7753,7 +7747,7 @@ static void update_intel_pt_cfg(struct kvm_vcpu *vcpu)
> vmx->pt_desc.ctl_bitmask &= ~(0xfULL << (32 + i * 4));
> }
>
> -static void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
> +void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -7907,7 +7901,7 @@ static __init void vmx_set_cpu_caps(void)
> kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG);
> }
>
> -static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
> +void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
> {
> to_vmx(vcpu)->req_immediate_exit = true;
> }
> @@ -7946,10 +7940,10 @@ static int vmx_check_intercept_io(struct kvm_vcpu *vcpu,
> return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
> }
>
> -static int vmx_check_intercept(struct kvm_vcpu *vcpu,
> - struct x86_instruction_info *info,
> - enum x86_intercept_stage stage,
> - struct x86_exception *exception)
> +int vmx_check_intercept(struct kvm_vcpu *vcpu,
> + struct x86_instruction_info *info,
> + enum x86_intercept_stage stage,
> + struct x86_exception *exception)
> {
> struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
>
> @@ -8029,8 +8023,8 @@ static inline int u64_shl_div_u64(u64 a, unsigned int shift,
> return 0;
> }
>
> -static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
> - bool *expired)
> +int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
> + bool *expired)
> {
> struct vcpu_vmx *vmx;
> u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
> @@ -8069,13 +8063,13 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
> return 0;
> }
>
> -static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
> +void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
> {
> to_vmx(vcpu)->hv_deadline_tsc = -1;
> }
> #endif
>
> -static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
> +void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
> {
> if (!kvm_pause_in_guest(vcpu->kvm))
> shrink_ple_window(vcpu);
> @@ -8104,7 +8098,7 @@ void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
> secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML);
> }
>
> -static void vmx_setup_mce(struct kvm_vcpu *vcpu)
> +void vmx_setup_mce(struct kvm_vcpu *vcpu)
> {
> if (vcpu->arch.mcg_cap & MCG_LMCE_P)
> to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
> @@ -8115,7 +8109,7 @@ static void vmx_setup_mce(struct kvm_vcpu *vcpu)
> }
>
> #ifdef CONFIG_KVM_SMM
> -static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> +int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> {
> /* we need a nested vmexit to enter SMM, postpone if run is pending */
> if (to_vmx(vcpu)->nested.nested_run_pending)
> @@ -8123,7 +8117,7 @@ static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
> return !is_smm(vcpu);
> }
>
> -static int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
> +int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
>
> @@ -8144,7 +8138,7 @@ static int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
> return 0;
> }
>
> -static int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
> +int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
> {
> struct vcpu_vmx *vmx = to_vmx(vcpu);
> int ret;
> @@ -8165,18 +8159,18 @@ static int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
> return 0;
> }
>
> -static void vmx_enable_smi_window(struct kvm_vcpu *vcpu)
> +void vmx_enable_smi_window(struct kvm_vcpu *vcpu)
> {
> /* RSM will cause a vmexit anyway. */
> }
> #endif
>
> -static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
> +bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
> {
> return to_vmx(vcpu)->nested.vmxon && !is_guest_mode(vcpu);
> }
>
> -static void vmx_migrate_timers(struct kvm_vcpu *vcpu)
> +void vmx_migrate_timers(struct kvm_vcpu *vcpu)
> {
> if (is_guest_mode(vcpu)) {
> struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer;
> @@ -8186,7 +8180,7 @@ static void vmx_migrate_timers(struct kvm_vcpu *vcpu)
> }
> }
>
> -static void vmx_hardware_unsetup(void)
> +void vmx_hardware_unsetup(void)
> {
> kvm_set_posted_intr_wakeup_handler(NULL);
>
> @@ -8196,18 +8190,7 @@ static void vmx_hardware_unsetup(void)
> free_kvm_area();
> }
>
> -#define VMX_REQUIRED_APICV_INHIBITS \
> -( \
> - BIT(APICV_INHIBIT_REASON_DISABLE)| \
> - BIT(APICV_INHIBIT_REASON_ABSENT) | \
> - BIT(APICV_INHIBIT_REASON_HYPERV) | \
> - BIT(APICV_INHIBIT_REASON_BLOCKIRQ) | \
> - BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) | \
> - BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) | \
> - BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED) \
> -)
> -
> -static void vmx_vm_destroy(struct kvm *kvm)
> +void vmx_vm_destroy(struct kvm *kvm)
> {
> struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
>
> @@ -8258,151 +8241,6 @@ gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags
> return (sign_extend64(gva, lam_bit) & ~BIT_ULL(63)) | (gva & BIT_ULL(63));
> }
>
> -static struct kvm_x86_ops vmx_x86_ops __initdata = {
> - .name = KBUILD_MODNAME,
> -
> - .check_processor_compatibility = vmx_check_processor_compat,
> -
> - .hardware_unsetup = vmx_hardware_unsetup,
> -
> - .hardware_enable = vmx_hardware_enable,
> - .hardware_disable = vmx_hardware_disable,
> - .has_emulated_msr = vmx_has_emulated_msr,
> -
> - .is_vm_type_supported = vmx_is_vm_type_supported,
> - .vm_size = sizeof(struct kvm_vmx),
> - .vm_init = vmx_vm_init,
> - .vm_destroy = vmx_vm_destroy,
> -
> - .vcpu_precreate = vmx_vcpu_precreate,
> - .vcpu_create = vmx_vcpu_create,
> - .vcpu_free = vmx_vcpu_free,
> - .vcpu_reset = vmx_vcpu_reset,
> -
> - .prepare_switch_to_guest = vmx_prepare_switch_to_guest,
> - .vcpu_load = vmx_vcpu_load,
> - .vcpu_put = vmx_vcpu_put,
> -
> - .update_exception_bitmap = vmx_update_exception_bitmap,
> - .get_msr_feature = vmx_get_msr_feature,
> - .get_msr = vmx_get_msr,
> - .set_msr = vmx_set_msr,
> - .get_segment_base = vmx_get_segment_base,
> - .get_segment = vmx_get_segment,
> - .set_segment = vmx_set_segment,
> - .get_cpl = vmx_get_cpl,
> - .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
> - .is_valid_cr0 = vmx_is_valid_cr0,
> - .set_cr0 = vmx_set_cr0,
> - .is_valid_cr4 = vmx_is_valid_cr4,
> - .set_cr4 = vmx_set_cr4,
> - .set_efer = vmx_set_efer,
> - .get_idt = vmx_get_idt,
> - .set_idt = vmx_set_idt,
> - .get_gdt = vmx_get_gdt,
> - .set_gdt = vmx_set_gdt,
> - .set_dr7 = vmx_set_dr7,
> - .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
> - .cache_reg = vmx_cache_reg,
> - .get_rflags = vmx_get_rflags,
> - .set_rflags = vmx_set_rflags,
> - .get_if_flag = vmx_get_if_flag,
> -
> - .flush_tlb_all = vmx_flush_tlb_all,
> - .flush_tlb_current = vmx_flush_tlb_current,
> - .flush_tlb_gva = vmx_flush_tlb_gva,
> - .flush_tlb_guest = vmx_flush_tlb_guest,
> -
> - .vcpu_pre_run = vmx_vcpu_pre_run,
> - .vcpu_run = vmx_vcpu_run,
> - .handle_exit = vmx_handle_exit,
> - .skip_emulated_instruction = vmx_skip_emulated_instruction,
> - .update_emulated_instruction = vmx_update_emulated_instruction,
> - .set_interrupt_shadow = vmx_set_interrupt_shadow,
> - .get_interrupt_shadow = vmx_get_interrupt_shadow,
> - .patch_hypercall = vmx_patch_hypercall,
> - .inject_irq = vmx_inject_irq,
> - .inject_nmi = vmx_inject_nmi,
> - .inject_exception = vmx_inject_exception,
> - .cancel_injection = vmx_cancel_injection,
> - .interrupt_allowed = vmx_interrupt_allowed,
> - .nmi_allowed = vmx_nmi_allowed,
> - .get_nmi_mask = vmx_get_nmi_mask,
> - .set_nmi_mask = vmx_set_nmi_mask,
> - .enable_nmi_window = vmx_enable_nmi_window,
> - .enable_irq_window = vmx_enable_irq_window,
> - .update_cr8_intercept = vmx_update_cr8_intercept,
> - .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
> - .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
> - .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
> - .load_eoi_exitmap = vmx_load_eoi_exitmap,
> - .apicv_pre_state_restore = vmx_apicv_pre_state_restore,
> - .required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS,
> - .hwapic_irr_update = vmx_hwapic_irr_update,
> - .hwapic_isr_update = vmx_hwapic_isr_update,
> - .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
> - .sync_pir_to_irr = vmx_sync_pir_to_irr,
> - .deliver_interrupt = vmx_deliver_interrupt,
> - .dy_apicv_has_pending_interrupt = pi_has_pending_interrupt,
> -
> - .set_tss_addr = vmx_set_tss_addr,
> - .set_identity_map_addr = vmx_set_identity_map_addr,
> - .get_mt_mask = vmx_get_mt_mask,
> -
> - .get_exit_info = vmx_get_exit_info,
> -
> - .vcpu_after_set_cpuid = vmx_vcpu_after_set_cpuid,
> -
> - .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
> -
> - .get_l2_tsc_offset = vmx_get_l2_tsc_offset,
> - .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier,
> - .write_tsc_offset = vmx_write_tsc_offset,
> - .write_tsc_multiplier = vmx_write_tsc_multiplier,
> -
> - .load_mmu_pgd = vmx_load_mmu_pgd,
> -
> - .check_intercept = vmx_check_intercept,
> - .handle_exit_irqoff = vmx_handle_exit_irqoff,
> -
> - .request_immediate_exit = vmx_request_immediate_exit,
> -
> - .sched_in = vmx_sched_in,
> -
> - .cpu_dirty_log_size = PML_ENTITY_NUM,
> - .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging,
> -
> - .nested_ops = &vmx_nested_ops,
> -
> - .pi_update_irte = vmx_pi_update_irte,
> - .pi_start_assignment = vmx_pi_start_assignment,
> -
> -#ifdef CONFIG_X86_64
> - .set_hv_timer = vmx_set_hv_timer,
> - .cancel_hv_timer = vmx_cancel_hv_timer,
> -#endif
> -
> - .setup_mce = vmx_setup_mce,
> -
> -#ifdef CONFIG_KVM_SMM
> - .smi_allowed = vmx_smi_allowed,
> - .enter_smm = vmx_enter_smm,
> - .leave_smm = vmx_leave_smm,
> - .enable_smi_window = vmx_enable_smi_window,
> -#endif
> -
> - .check_emulate_instruction = vmx_check_emulate_instruction,
> - .apic_init_signal_blocked = vmx_apic_init_signal_blocked,
> - .migrate_timers = vmx_migrate_timers,
> -
> - .msr_filter_changed = vmx_msr_filter_changed,
> - .complete_emulated_msr = kvm_complete_insn_gp,
> -
> - .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
> -
> - .get_untagged_addr = vmx_get_untagged_addr,
> -};
> -
> static unsigned int vmx_handle_intel_pt_intr(void)
> {
> struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
> @@ -8468,9 +8306,7 @@ static void __init vmx_setup_me_spte_mask(void)
> kvm_mmu_set_me_spte_mask(0, me_mask);
> }
>
> -static struct kvm_x86_init_ops vmx_init_ops __initdata;
> -
> -static __init int hardware_setup(void)
> +__init int vmx_hardware_setup(void)
> {
> unsigned long host_bndcfgs;
> struct desc_ptr dt;
> @@ -8539,16 +8375,16 @@ static __init int hardware_setup(void)
> * using the APIC_ACCESS_ADDR VMCS field.
> */
> if (!flexpriority_enabled)
> - vmx_x86_ops.set_apic_access_page_addr = NULL;
> + vt_x86_ops.set_apic_access_page_addr = NULL;
>
> if (!cpu_has_vmx_tpr_shadow())
> - vmx_x86_ops.update_cr8_intercept = NULL;
> + vt_x86_ops.update_cr8_intercept = NULL;
>
> #if IS_ENABLED(CONFIG_HYPERV)
> if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH
> && enable_ept) {
> - vmx_x86_ops.flush_remote_tlbs = hv_flush_remote_tlbs;
> - vmx_x86_ops.flush_remote_tlbs_range = hv_flush_remote_tlbs_range;
> + vt_x86_ops.flush_remote_tlbs = hv_flush_remote_tlbs;
> + vt_x86_ops.flush_remote_tlbs_range = hv_flush_remote_tlbs_range;
> }
> #endif
>
> @@ -8563,7 +8399,7 @@ static __init int hardware_setup(void)
> if (!cpu_has_vmx_apicv())
> enable_apicv = 0;
> if (!enable_apicv)
> - vmx_x86_ops.sync_pir_to_irr = NULL;
> + vt_x86_ops.sync_pir_to_irr = NULL;
>
> if (!enable_apicv || !cpu_has_vmx_ipiv())
> enable_ipiv = false;
> @@ -8599,7 +8435,7 @@ static __init int hardware_setup(void)
> enable_pml = 0;
>
> if (!enable_pml)
> - vmx_x86_ops.cpu_dirty_log_size = 0;
> + vt_x86_ops.cpu_dirty_log_size = 0;
>
> if (!cpu_has_vmx_preemption_timer())
> enable_preemption_timer = false;
> @@ -8624,9 +8460,9 @@ static __init int hardware_setup(void)
> }
>
> if (!enable_preemption_timer) {
> - vmx_x86_ops.set_hv_timer = NULL;
> - vmx_x86_ops.cancel_hv_timer = NULL;
> - vmx_x86_ops.request_immediate_exit = __kvm_request_immediate_exit;
> + vt_x86_ops.set_hv_timer = NULL;
> + vt_x86_ops.cancel_hv_timer = NULL;
> + vt_x86_ops.request_immediate_exit = __kvm_request_immediate_exit;
> }
>
> kvm_caps.supported_mce_cap |= MCG_LMCE_P;
> @@ -8637,9 +8473,9 @@ static __init int hardware_setup(void)
> if (!enable_ept || !enable_pmu || !cpu_has_vmx_intel_pt())
> pt_mode = PT_MODE_SYSTEM;
> if (pt_mode == PT_MODE_HOST_GUEST)
> - vmx_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr;
> + vt_init_ops.handle_intel_pt_intr = vmx_handle_intel_pt_intr;
> else
> - vmx_init_ops.handle_intel_pt_intr = NULL;
> + vt_init_ops.handle_intel_pt_intr = NULL;
>
> setup_default_sgx_lepubkeyhash();
>
> @@ -8662,14 +8498,6 @@ static __init int hardware_setup(void)
> return r;
> }
>
> -static struct kvm_x86_init_ops vmx_init_ops __initdata = {
> - .hardware_setup = hardware_setup,
> - .handle_intel_pt_intr = NULL,
> -
> - .runtime_ops = &vmx_x86_ops,
> - .pmu_ops = &intel_pmu_ops,
> -};
> -
> static void vmx_cleanup_l1d_flush(void)
> {
> if (vmx_l1d_flush_pages) {
> @@ -8711,7 +8539,7 @@ static int __init vmx_init(void)
> */
> hv_init_evmcs();
>
> - r = kvm_x86_vendor_init(&vmx_init_ops);
> + r = kvm_x86_vendor_init(&vt_init_ops);
> if (r)
> return r;
>
> diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h
> new file mode 100644
> index 000000000000..b6836bedc4d3
> --- /dev/null
> +++ b/arch/x86/kvm/vmx/x86_ops.h
> @@ -0,0 +1,125 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __KVM_X86_VMX_X86_OPS_H
> +#define __KVM_X86_VMX_X86_OPS_H
> +
> +#include <linux/kvm_host.h>
> +
> +#include "x86.h"
> +
> +__init int vmx_hardware_setup(void);
> +
> +extern struct kvm_x86_ops vt_x86_ops __initdata;
> +extern struct kvm_x86_init_ops vt_init_ops __initdata;
> +
> +void vmx_hardware_unsetup(void);
> +int vmx_check_processor_compat(void);
> +int vmx_hardware_enable(void);
> +void vmx_hardware_disable(void);
> +bool vmx_is_vm_type_supported(unsigned long type);
> +int vmx_vm_init(struct kvm *kvm);
> +void vmx_vm_destroy(struct kvm *kvm);
> +int vmx_vcpu_precreate(struct kvm *kvm);
> +int vmx_vcpu_create(struct kvm_vcpu *vcpu);
> +int vmx_vcpu_pre_run(struct kvm_vcpu *vcpu);
> +fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu);
> +void vmx_vcpu_free(struct kvm_vcpu *vcpu);
> +void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event);
> +void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
> +void vmx_vcpu_put(struct kvm_vcpu *vcpu);
> +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath);
> +void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu);
> +int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu);
> +void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu);
> +int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info);
> +#ifdef CONFIG_KVM_SMM
> +int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection);
> +int vmx_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram);
> +int vmx_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram);
> +void vmx_enable_smi_window(struct kvm_vcpu *vcpu);
> +#endif
> +int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
> + void *insn, int insn_len);
> +int vmx_check_intercept(struct kvm_vcpu *vcpu,
> + struct x86_instruction_info *info,
> + enum x86_intercept_stage stage,
> + struct x86_exception *exception);
> +bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu);
> +void vmx_migrate_timers(struct kvm_vcpu *vcpu);
> +void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu);
> +void vmx_apicv_pre_state_restore(struct kvm_vcpu *vcpu);
> +bool vmx_check_apicv_inhibit_reasons(enum kvm_apicv_inhibit reason);
> +void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr);
> +void vmx_hwapic_isr_update(int max_isr);
> +bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu);
> +int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu);
> +void vmx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
> + int trig_mode, int vector);
> +void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu);
> +bool vmx_has_emulated_msr(struct kvm *kvm, u32 index);
> +void vmx_msr_filter_changed(struct kvm_vcpu *vcpu);
> +void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu);
> +void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu);
> +int vmx_get_msr_feature(struct kvm_msr_entry *msr);
> +int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info);
> +u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg);
> +void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
> +void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
> +int vmx_get_cpl(struct kvm_vcpu *vcpu);
> +void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l);
> +bool vmx_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
> +void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
> +void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level);
> +void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
> +bool vmx_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
> +int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer);
> +void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
> +void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
> +void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
> +void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
> +void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val);
> +void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu);
> +void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg);
> +unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu);
> +void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
> +bool vmx_get_if_flag(struct kvm_vcpu *vcpu);
> +void vmx_flush_tlb_all(struct kvm_vcpu *vcpu);
> +void vmx_flush_tlb_current(struct kvm_vcpu *vcpu);
> +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr);
> +void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu);
> +void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask);
> +u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu);
> +void vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall);
> +void vmx_inject_irq(struct kvm_vcpu *vcpu, bool reinjected);
> +void vmx_inject_nmi(struct kvm_vcpu *vcpu);
> +void vmx_inject_exception(struct kvm_vcpu *vcpu);
> +void vmx_cancel_injection(struct kvm_vcpu *vcpu);
> +int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection);
> +int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection);
> +bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
> +void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
> +void vmx_enable_nmi_window(struct kvm_vcpu *vcpu);
> +void vmx_enable_irq_window(struct kvm_vcpu *vcpu);
> +void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr);
> +void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu);
> +void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu);
> +void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap);
> +int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
> +int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr);
> +u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
> +void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
> + u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code);
> +u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu);
> +u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
> +void vmx_write_tsc_offset(struct kvm_vcpu *vcpu);
> +void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu);
> +void vmx_request_immediate_exit(struct kvm_vcpu *vcpu);
> +void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu);
> +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu);
> +#ifdef CONFIG_X86_64
> +int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
> + bool *expired);
> +void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu);
> +#endif
> +void vmx_setup_mce(struct kvm_vcpu *vcpu);
> +
> +#endif /* __KVM_X86_VMX_X86_OPS_H */
> --
> 2.25.1
>
>
