When the guest indicates that the TLB doesn't need to be flushed in a
CR3 switch, we can also skip resyncing the shadow page tables since an
out-of-sync shadow page table is equivalent to an out-of-sync TLB.
Signed-off-by: Junaid Shahid <junaids@xxxxxxxxxx>
---
arch/x86/kvm/mmu.c | 38 +++++++++++++++++++++++++++++++++-----
arch/x86/kvm/vmx.c | 11 ++++++-----
arch/x86/kvm/x86.c | 6 +++---
3 files changed, 42 insertions(+), 13 deletions(-)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 4464565eec96..5b26c191052c 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -4087,9 +4087,19 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3,
*/
kvm_make_request(KVM_REQ_LOAD_CR3, vcpu);
- kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
- if (!skip_tlb_flush)
+ if (!skip_tlb_flush) {
+ kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
kvm_x86_ops->tlb_flush(vcpu, true);
+ }
+
+ /*
+ * The last MMIO access's GVA and GPA are cached in the
+ * VCPU. When switching to a new CR3, that GVA->GPA
+ * mapping may no longer be valid. So clear any cached
+ * MMIO info even when we don't need to sync the shadow
+ * page tables.
+ */
+ vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
__clear_sp_write_flooding_count(
page_header(mmu->root_hpa));
@@ -5205,6 +5215,21 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
struct kvm_mmu *mmu = &vcpu->arch.mmu;
mmu->invlpg(vcpu, gva, mmu->root_hpa);
+
+ /*
+ * INVLPG is required to invalidate any global mappings for the VA,
+ * irrespective of PCID. Since it would take us roughly similar amount
+ * of work to determine whether the prev_root mapping of the VA is
+ * marked global, or to just sync it blindly, so we might as well just
+ * always sync it.
+ *
+ * Mappings not reachable via the current cr3 or the prev_root.cr3 will
+ * be synced when switching to that cr3, so nothing needs to be done
+ * here for them.
+ */
+ if (VALID_PAGE(mmu->prev_root.hpa))
+ mmu->invlpg(vcpu, gva, mmu->prev_root.hpa);
+
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
++vcpu->stat.invlpg;
}
@@ -5220,14 +5245,17 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
}
if (VALID_PAGE(mmu->prev_root.hpa) &&
- pcid == kvm_get_pcid(vcpu, mmu->prev_root.cr3))
+ pcid == kvm_get_pcid(vcpu, mmu->prev_root.cr3)) {
+ mmu->invlpg(vcpu, gva, mmu->prev_root.hpa);
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+ }
++vcpu->stat.invlpg;
/*
- * Mappings not reachable via the current cr3 will be synced when
- * switching to that cr3, so nothing needs to be synced here for them.
+ * Mappings not reachable via the current cr3 or the prev_root.cr3 will
+ * be synced when switching to that cr3, so nothing needs to be done
+ * here for them.
*/
}
EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva);
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index ac4e78c2da38..98b2a9ca3ba9 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -8682,14 +8682,15 @@ static int handle_invpcid(struct kvm_vcpu *vcpu)
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
}
- if (kvm_get_pcid(vcpu, vcpu->arch.mmu.prev_root.cr3)
+ if (kvm_get_pcid(vcpu, vcpu->arch.mmu.prev_root.cr3)
== operand.pcid)
- kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+ kvm_mmu_free_roots(vcpu, KVM_MMU_ROOT_PREVIOUS);
+
/*
- * If the current cr3 does not use the given PCID, then nothing
- * needs to be synced here because a resync will happen anyway
- * before switching to any other CR3.
+ * If neither the current cr3 nor the prev_root.cr3 use the
+ * given PCID, then nothing needs to be done here because a
+ * resync will happen anyway before switching to any other CR3.
*/
skip_emulated_instruction(vcpu);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 151a8f61c7f1..6a179f54b42c 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -858,10 +858,10 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
#endif
if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
- kvm_mmu_sync_roots(vcpu);
-
- if (!skip_tlb_flush)
+ if (!skip_tlb_flush) {
+ kvm_mmu_sync_roots(vcpu);
kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
+ }
return 0;
}
--
2.18.0.rc1.242.g61856ae69a-goog
