----- junaids@xxxxxxxxxx wrote:
> Add a parameter to the set_cr3 handlers to allow skipping the TLB
> flush.
>
> Signed-off-by: Junaid Shahid <junaids@xxxxxxxxxx>
> ---
> arch/x86/include/asm/kvm_host.h | 9 ++++++---
> arch/x86/kvm/mmu.c | 4 ++--
> arch/x86/kvm/svm.c | 17 +++++++++++------
> arch/x86/kvm/vmx.c | 7 +++++--
> 4 files changed, 24 insertions(+), 13 deletions(-)
>
> diff --git a/arch/x86/include/asm/kvm_host.h
> b/arch/x86/include/asm/kvm_host.h
> index e9c56f21ac3c..21eb513205b0 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -330,7 +330,8 @@ struct rsvd_bits_validate {
> * current mmu mode.
> */
> struct kvm_mmu {
> - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
> + void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root,
> + bool skip_tlb_flush);
> unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
> u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
> int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
> @@ -957,7 +958,8 @@ struct kvm_x86_ops {
> void (*decache_cr3)(struct kvm_vcpu *vcpu);
> void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
> void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
> - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
> + void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3,
> + bool skip_tlb_flush);
> int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
> void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
> void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
> @@ -1009,7 +1011,8 @@ struct kvm_x86_ops {
> bool (*rdtscp_supported)(void);
> bool (*invpcid_supported)(void);
>
> - void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
> + void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3,
> + bool skip_tlb_flush);
>
> void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2
> *entry);
>
> diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
> index 0107129d8ed9..611ecc37e5d2 100644
> --- a/arch/x86/kvm/mmu.c
> +++ b/arch/x86/kvm/mmu.c
> @@ -4048,7 +4048,7 @@ static bool fast_cr3_switch(struct kvm_vcpu
> *vcpu, gpa_t old_cr3)
> kvm_mmu_sync_roots(vcpu);
> __clear_sp_write_flooding_count(
> page_header(mmu->root_hpa));
> - mmu->set_cr3(vcpu, mmu->root_hpa);
> + mmu->set_cr3(vcpu, mmu->root_hpa, false);
> return true;
> }
> }
> @@ -4774,7 +4774,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
> if (r)
> goto out;
> /* set_cr3() should ensure TLB has been flushed */
> - vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
> + vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa, false);
> out:
> return r;
> }
> diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
> index b58787daf9f8..fd8b0dc5136f 100644
> --- a/arch/x86/kvm/svm.c
> +++ b/arch/x86/kvm/svm.c
> @@ -2855,13 +2855,14 @@ static u64 nested_svm_get_tdp_pdptr(struct
> kvm_vcpu *vcpu, int index)
> }
>
> static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
> - unsigned long root)
> + unsigned long root, bool skip_tlb_flush)
> {
> struct vcpu_svm *svm = to_svm(vcpu);
>
> svm->vmcb->control.nested_cr3 = __sme_set(root);
> mark_dirty(svm->vmcb, VMCB_NPT);
> - svm_flush_tlb(vcpu, true);
> + if (!skip_tlb_flush)
> + svm_flush_tlb(vcpu, true);
> }
>
> static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
> @@ -5727,16 +5728,19 @@ static void svm_vcpu_run(struct kvm_vcpu
> *vcpu)
> }
> STACK_FRAME_NON_STANDARD(svm_vcpu_run);
>
> -static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
> +static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root,
> + bool skip_tlb_flush)
> {
> struct vcpu_svm *svm = to_svm(vcpu);
>
> svm->vmcb->save.cr3 = __sme_set(root);
> mark_dirty(svm->vmcb, VMCB_CR);
> - svm_flush_tlb(vcpu, true);
> + if (!skip_tlb_flush)
> + svm_flush_tlb(vcpu, true);
> }
>
> -static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
> +static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root,
> + bool skip_tlb_flush)
> {
> struct vcpu_svm *svm = to_svm(vcpu);
>
> @@ -5747,7 +5751,8 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu,
> unsigned long root)
> svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
> mark_dirty(svm->vmcb, VMCB_CR);
>
> - svm_flush_tlb(vcpu, true);
> + if (!skip_tlb_flush)
> + svm_flush_tlb(vcpu, true);
> }
>
> static int is_disabled(void)
> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index aafcc9881e88..7a73e97bae61 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -4738,7 +4738,8 @@ static u64 construct_eptp(struct kvm_vcpu *vcpu,
> unsigned long root_hpa)
> return eptp;
> }
>
> -static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
> +static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
> + bool skip_tlb_flush)
> {
> unsigned long guest_cr3;
> u64 eptp;
> @@ -4755,7 +4756,9 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu,
> unsigned long cr3)
> ept_load_pdptrs(vcpu);
> }
>
> - vmx_flush_tlb(vcpu, true);
> + if (!skip_tlb_flush)
> + vmx_flush_tlb(vcpu, true);
> +
> vmcs_writel(GUEST_CR3, guest_cr3);
> }
>
> --
> 2.17.0.441.gb46fe60e1d-goog
Reviewed-by: Liran Alon <liran.alon@xxxxxxxxxx>
