[PATCH 05/11] kvm: x86: Add ability to skip TLB flush when switching CR3

[Junaid Shahid <junaids@xxxxxxxxxx>]

Add a parameter to the set_cr3 handlers to allow skipping the TLB flush.

Signed-off-by: Junaid Shahid <junaids@xxxxxxxxxx>
---
 arch/x86/include/asm/kvm_host.h |  9 ++++++---
 arch/x86/kvm/mmu.c              |  4 ++--
 arch/x86/kvm/svm.c              | 17 +++++++++++------
 arch/x86/kvm/vmx.c              |  7 +++++--
 4 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index e9c56f21ac3c..21eb513205b0 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -330,7 +330,8 @@ struct rsvd_bits_validate {
  * current mmu mode.
  */
 struct kvm_mmu {
-	void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root);
+	void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long root,
+			bool skip_tlb_flush);
 	unsigned long (*get_cr3)(struct kvm_vcpu *vcpu);
 	u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
 	int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err,
@@ -957,7 +958,8 @@ struct kvm_x86_ops {
 	void (*decache_cr3)(struct kvm_vcpu *vcpu);
 	void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu);
 	void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0);
-	void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
+	void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3,
+			bool skip_tlb_flush);
 	int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4);
 	void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer);
 	void (*get_idt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
@@ -1009,7 +1011,8 @@ struct kvm_x86_ops {
 	bool (*rdtscp_supported)(void);
 	bool (*invpcid_supported)(void);
 
-	void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3);
+	void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3,
+			    bool skip_tlb_flush);
 
 	void (*set_supported_cpuid)(u32 func, struct kvm_cpuid_entry2 *entry);
 
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 0107129d8ed9..611ecc37e5d2 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4048,7 +4048,7 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t old_cr3)
 			kvm_mmu_sync_roots(vcpu);
 			__clear_sp_write_flooding_count(
 						page_header(mmu->root_hpa));
-			mmu->set_cr3(vcpu, mmu->root_hpa);
+			mmu->set_cr3(vcpu, mmu->root_hpa, false);
 			return true;
 		}
 	}
@@ -4774,7 +4774,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu)
 	if (r)
 		goto out;
 	/* set_cr3() should ensure TLB has been flushed */
-	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
+	vcpu->arch.mmu.set_cr3(vcpu, vcpu->arch.mmu.root_hpa, false);
 out:
 	return r;
 }
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index b58787daf9f8..fd8b0dc5136f 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -2855,13 +2855,14 @@ static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
 }
 
 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
-				   unsigned long root)
+				   unsigned long root, bool skip_tlb_flush)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
 	svm->vmcb->control.nested_cr3 = __sme_set(root);
 	mark_dirty(svm->vmcb, VMCB_NPT);
-	svm_flush_tlb(vcpu, true);
+	if (!skip_tlb_flush)
+		svm_flush_tlb(vcpu, true);
 }
 
 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
@@ -5727,16 +5728,19 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 }
 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
 
-static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
+static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root,
+			bool skip_tlb_flush)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
 	svm->vmcb->save.cr3 = __sme_set(root);
 	mark_dirty(svm->vmcb, VMCB_CR);
-	svm_flush_tlb(vcpu, true);
+	if (!skip_tlb_flush)
+		svm_flush_tlb(vcpu, true);
 }
 
-static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
+static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root,
+			bool skip_tlb_flush)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -5747,7 +5751,8 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
 	svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
 	mark_dirty(svm->vmcb, VMCB_CR);
 
-	svm_flush_tlb(vcpu, true);
+	if (!skip_tlb_flush)
+		svm_flush_tlb(vcpu, true);
 }
 
 static int is_disabled(void)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index aafcc9881e88..7a73e97bae61 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -4738,7 +4738,8 @@ static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa)
 	return eptp;
 }
 
-static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
+static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
+			bool skip_tlb_flush)
 {
 	unsigned long guest_cr3;
 	u64 eptp;
@@ -4755,7 +4756,9 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 		ept_load_pdptrs(vcpu);
 	}
 
-	vmx_flush_tlb(vcpu, true);
+	if (!skip_tlb_flush)
+		vmx_flush_tlb(vcpu, true);
+
 	vmcs_writel(GUEST_CR3, guest_cr3);
 }
 
-- 
2.17.0.441.gb46fe60e1d-goog