When the guest indicates that the TLB doesn't need to be flushed in a
CR3 switch, we can also skip resyncing the shadow page tables since an
out-of-sync shadow page table is equivalent to an out-of-sync TLB.
Signed-off-by: Junaid Shahid <junaids@xxxxxxxxxx>
---
arch/x86/kvm/mmu.c | 39 ++++++++++++++++++++++++++++++++++++---
arch/x86/kvm/vmx.c | 9 ++++++---
arch/x86/kvm/x86.c | 6 +++---
3 files changed, 45 insertions(+), 9 deletions(-)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 8b3229f8ae71..160dc84c15be 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4068,7 +4068,18 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t old_cr3,
* have set here and allocate a new one.
*/
- kvm_mmu_sync_roots(vcpu);
+ if (!skip_tlb_flush)
+ kvm_mmu_sync_roots(vcpu);
+
+ /*
+ * The last MMIO access's GVA and GPA are cached in the
+ * VCPU. When switching to a new CR3, that GVA->GPA
+ * mapping may no longer be valid. So clear any cached
+ * MMIO info even when we don't need to sync the shadow
+ * page tables.
+ */
+ vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
+
__clear_sp_write_flooding_count(
page_header(mmu->root_hpa));
mmu->set_cr3(vcpu, mmu->root_hpa | pcid,
@@ -5140,6 +5151,21 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
struct kvm_mmu *mmu = &vcpu->arch.mmu;
mmu->invlpg(vcpu, gva, mmu->root_hpa);
+
+ /*
+ * INVLPG is required to invalidate any global mappings for the VA,
+ * irrespective of PCID. Since it would take us roughly similar amount
+ * of work to determine whether the prev_cr3 mapping of the VA is marked
+ * global, or to just sync it blindly, so we might as well just always
+ * sync it.
+ *
+ * Mappings not reachable via the current cr3 or the prev_cr3 will be
+ * synced when switching to that cr3, so nothing needs to be done here
+ * for them.
+ */
+ if (VALID_PAGE(mmu->prev_root_hpa))
+ mmu->invlpg(vcpu, gva, mmu->prev_root_hpa);
+
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
++vcpu->stat.invlpg;
}
@@ -5154,11 +5180,18 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
}
+ if (VALID_PAGE(mmu->prev_root_hpa) &&
+ pcid == kvm_get_pcid(vcpu, mmu->prev_cr3)) {
+ mmu->invlpg(vcpu, gva, mmu->prev_root_hpa);
+ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+ }
+
++vcpu->stat.invlpg;
/*
- * Mappings not reachable via the current cr3 will be synced when
- * switching to that cr3, so nothing needs to be done here for them.
+ * Mappings not reachable via the current cr3 or the prev_cr3 will be
+ * synced when switching to that cr3, so nothing needs to be done here
+ * for them.
*/
}
EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva);
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 234498e0000f..b11ec063564f 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -8422,10 +8422,13 @@ static int handle_invpcid(struct kvm_vcpu *vcpu)
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
}
+ if (kvm_get_pcid(vcpu, vcpu->arch.mmu.prev_cr3) == operand.pcid)
+ kvm_mmu_free_roots(vcpu, KVM_MMU_ROOT_PREVIOUS);
+
/*
- * If the current cr3 does not use the given PCID, then nothing
- * needs to be done here because a resync will happen anyway
- * before switching to any other CR3.
+ * If neither the current cr3 nor the prev_cr3 use the given
+ * PCID, then nothing needs to be done here because a resync
+ * will happen anyway before switching to any other CR3.
*/
skip_emulated_instruction(vcpu);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index baeb8447ede2..7b7da35f1ea6 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -855,10 +855,10 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
#endif
if (cr3 == old_cr3 && !pdptrs_changed(vcpu)) {
- kvm_mmu_sync_roots(vcpu);
-
- if (!skip_tlb_flush)
+ if (!skip_tlb_flush) {
+ kvm_mmu_sync_roots(vcpu);
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+ }
return 0;
}
--
2.17.0.441.gb46fe60e1d-goog
