From: David Woodhouse <dwmw@xxxxxxxxxxxx>
The busy loop in hva_to_pfn_retry() is worse than a normal page fault
retry loop because it spins even while it's waiting for the invalidation
to complete. It isn't just that a page might get faulted out again before
it's actually accessed.
Introduce a wait queue to be woken when kvm->mn_active_invalidate_count
reaches zero, and wait on it if there is any pending invalidation which
affects the GPC being refreshed.
Signed-off-by: David Woodhouse <dwmw@xxxxxxxxxxxx>
---
include/linux/kvm_host.h | 1 +
virt/kvm/kvm_main.c | 5 ++++-
virt/kvm/pfncache.c | 32 ++++++++++++++++++++++++++++----
3 files changed, 33 insertions(+), 5 deletions(-)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 1bfe2e8d52cd..a0739c343da5 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -772,6 +772,7 @@ struct kvm {
struct list_head gpc_list;
u64 mmu_gpc_invalidate_range_start;
u64 mmu_gpc_invalidate_range_end;
+ wait_queue_head_t gpc_invalidate_wq;
/*
* created_vcpus is protected by kvm->lock, and is incremented
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 84eb1ebb6f47..e04eb700448b 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -871,8 +871,10 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
* There can only be one waiter, since the wait happens under
* slots_lock.
*/
- if (wake)
+ if (wake) {
+ wake_up(&kvm->gpc_invalidate_wq);
rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
+ }
}
static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
@@ -1182,6 +1184,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
INIT_LIST_HEAD(&kvm->gpc_list);
spin_lock_init(&kvm->gpc_lock);
+ init_waitqueue_head(&kvm->gpc_invalidate_wq);
kvm->mmu_gpc_invalidate_range_start = KVM_HVA_ERR_BAD;
kvm->mmu_gpc_invalidate_range_end = KVM_HVA_ERR_BAD;
diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c
index eeb9bf43c04a..fa494eb3d924 100644
--- a/virt/kvm/pfncache.c
+++ b/virt/kvm/pfncache.c
@@ -135,13 +135,38 @@ static bool gpc_invalidations_pending(struct gfn_to_pfn_cache *gpc)
* No need for locking on GPC here because these fields are protected
* by gpc->refresh_lock.
*/
- guard(spinlock)(&gpc->kvm->mn_invalidate_lock);
-
return unlikely(gpc->kvm->mn_active_invalidate_count) &&
(gpc->kvm->mmu_gpc_invalidate_range_start <= gpc->uhva) &&
(gpc->kvm->mmu_gpc_invalidate_range_end > gpc->uhva);
}
+static bool gpc_wait_for_invalidations(struct gfn_to_pfn_cache *gpc)
+{
+ bool waited = false;
+
+ spin_lock(&gpc->kvm->mn_invalidate_lock);
+ if (gpc_invalidations_pending(gpc)) {
+ DEFINE_WAIT(wait);
+
+ waited = true;
+ for (;;) {
+ prepare_to_wait(&gpc->kvm->gpc_invalidate_wq, &wait,
+ TASK_UNINTERRUPTIBLE);
+
+ if (!gpc_invalidations_pending(gpc))
+ break;
+
+ spin_unlock(&gpc->kvm->mn_invalidate_lock);
+ schedule();
+ spin_lock(&gpc->kvm->mn_invalidate_lock);
+ }
+ finish_wait(&gpc->kvm->gpc_invalidate_wq, &wait);
+ }
+ spin_unlock(&gpc->kvm->mn_invalidate_lock);
+ return waited;
+}
+
+
static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
{
/* Note, the new page offset may be different than the old! */
@@ -191,8 +216,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
* to have been changted already, so hva_to_pfn() won't return
* a stale mapping in that case anyway.
*/
- while (gpc_invalidations_pending(gpc)) {
- cond_resched();
+ if (gpc_wait_for_invalidations(gpc)) {
write_lock_irq(&gpc->lock);
continue;
}
--
2.44.0
