On AMD CPUs without ensuring cache consistency, each memory page
reclamation in an SEV guest triggers a call to wbinvd_on_all_cpus(),
thereby affecting the performance of other programs on the host.
Typically, an AMD server may have 128 cores or more, while the SEV guest
might only utilize 8 of these cores. Meanwhile, host can use qemu-affinity
to bind these 8 vCPUs to specific physical CPUs.
Therefore, keeping a record of the physical core numbers each time a vCPU
runs can help avoid flushing the cache for all CPUs every time.
Suggested-by: Sean Christopherson <seanjc@xxxxxxxxxx>
Signed-off-by: Zheyun Shen <szy0127@xxxxxxxxxxx>
---
arch/x86/kvm/svm/sev.c | 39 ++++++++++++++++++++++++++++++++++++---
arch/x86/kvm/svm/svm.c | 2 ++
arch/x86/kvm/svm/svm.h | 5 ++++-
3 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index 1ce67de9d..91469edd1 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -252,6 +252,36 @@ static void sev_asid_free(struct kvm_sev_info *sev)
sev->misc_cg = NULL;
}
+static struct cpumask *sev_get_wbinvd_dirty_mask(struct kvm *kvm)
+{
+ struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+
+ return sev->wbinvd_dirty_mask;
+}
+
+void sev_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
+{
+ /*
+ * To optimize cache flushes when memory is reclaimed from an SEV VM,
+ * track physical CPUs that enter the guest for SEV VMs and thus can
+ * have encrypted, dirty data in the cache, and flush caches only for
+ * CPUs that have entered the guest.
+ */
+ cpumask_set_cpu(cpu, sev_get_wbinvd_dirty_mask(vcpu->kvm));
+}
+
+static void sev_do_wbinvd(struct kvm *kvm)
+{
+ struct cpumask *dirty_mask = sev_get_wbinvd_dirty_mask(kvm);
+
+ /*
+ * TODO: Clear CPUs from the bitmap prior to flushing. Doing so
+ * requires serializing multiple calls and having CPUs mark themselves
+ * "dirty" if they are currently running a vCPU for the VM.
+ */
+ wbinvd_on_many_cpus(dirty_mask);
+}
+
static void sev_decommission(unsigned int handle)
{
struct sev_data_decommission decommission;
@@ -448,6 +478,8 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp,
ret = sev_platform_init(&init_args);
if (ret)
goto e_free;
+ if (!zalloc_cpumask_var(&sev->wbinvd_dirty_mask, GFP_KERNEL_ACCOUNT))
+ goto e_free;
/* This needs to happen after SEV/SNP firmware initialization. */
if (vm_type == KVM_X86_SNP_VM) {
@@ -2778,7 +2810,7 @@ int sev_mem_enc_unregister_region(struct kvm *kvm,
* releasing the pages back to the system for use. CLFLUSH will
* not do this, so issue a WBINVD.
*/
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(kvm);
__unregister_enc_region_locked(kvm, region);
@@ -2926,6 +2958,7 @@ void sev_vm_destroy(struct kvm *kvm)
}
sev_asid_free(sev);
+ free_cpumask_var(sev->wbinvd_dirty_mask);
}
void __init sev_set_cpu_caps(void)
@@ -3129,7 +3162,7 @@ static void sev_flush_encrypted_page(struct kvm_vcpu *vcpu, void *va)
return;
do_wbinvd:
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(vcpu->kvm);
}
void sev_guest_memory_reclaimed(struct kvm *kvm)
@@ -3143,7 +3176,7 @@ void sev_guest_memory_reclaimed(struct kvm *kvm)
if (!sev_guest(kvm) || sev_snp_guest(kvm))
return;
- wbinvd_on_all_cpus();
+ sev_do_wbinvd(kvm);
}
void sev_free_vcpu(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 21dacd312..d2a423c0e 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -1565,6 +1565,8 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
}
if (kvm_vcpu_apicv_active(vcpu))
avic_vcpu_load(vcpu, cpu);
+ if (sev_guest(vcpu->kvm))
+ sev_vcpu_load(vcpu, cpu);
}
static void svm_vcpu_put(struct kvm_vcpu *vcpu)
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index 43fa6a16e..c8f42cb61 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -112,6 +112,8 @@ struct kvm_sev_info {
void *guest_req_buf; /* Bounce buffer for SNP Guest Request input */
void *guest_resp_buf; /* Bounce buffer for SNP Guest Request output */
struct mutex guest_req_mutex; /* Must acquire before using bounce buffers */
+ /* CPUs invoked VMRUN call wbinvd after guest memory is reclaimed */
+ struct cpumask *wbinvd_dirty_mask;
};
struct kvm_svm {
@@ -763,6 +765,7 @@ void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu);
int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end);
int sev_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn);
+void sev_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
#else
static inline struct page *snp_safe_alloc_page_node(int node, gfp_t gfp)
{
@@ -793,7 +796,7 @@ static inline int sev_private_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn)
{
return 0;
}
-
+static inline void sev_vcpu_load(struct kvm_vcpu *vcpu, int cpu) {}
#endif
/* vmenter.S */
--
2.34.1
