If guest access swapped out memory do not swap it in from vcpu thread
context. Setup slow work to do swapping and send async page fault to
a guest.
Allow async page fault injection only when guest is in user mode since
otherwise guest may be in non-sleepable context and will not be able to
reschedule.
Signed-off-by: Gleb Natapov <gleb@xxxxxxxxxx>
---
arch/x86/include/asm/kvm_host.h | 20 +++
arch/x86/kvm/mmu.c | 243 ++++++++++++++++++++++++++++++++++++++-
arch/x86/kvm/mmutrace.h | 60 ++++++++++
arch/x86/kvm/paging_tmpl.h | 16 +++-
arch/x86/kvm/x86.c | 22 +++-
5 files changed, 352 insertions(+), 9 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 2d1f526..e3cdbfe 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -15,6 +15,7 @@
#include <linux/mm.h>
#include <linux/mmu_notifier.h>
#include <linux/tracepoint.h>
+#include <linux/slow-work.h>
#include <linux/kvm.h>
#include <linux/kvm_para.h>
@@ -235,6 +236,18 @@ struct kvm_pv_mmu_op_buffer {
char buf[512] __aligned(sizeof(long));
};
+struct kvm_mmu_async_pf {
+ struct slow_work work;
+ struct list_head link;
+ struct kvm_vcpu *vcpu;
+ struct mm_struct *mm;
+ gva_t gva;
+ unsigned long addr;
+ u64 token;
+ struct page *page;
+ atomic_t used;
+};
+
struct kvm_pio_request {
unsigned long count;
int cur_count;
@@ -318,6 +331,11 @@ struct kvm_vcpu_arch {
unsigned long mmu_seq;
} update_pte;
+ struct list_head mmu_async_pf_done;
+ spinlock_t mmu_async_pf_lock;
+ struct kvm_mmu_async_pf *mmu_async_pf_work;
+ u32 async_pf_id;
+
struct i387_fxsave_struct host_fx_image;
struct i387_fxsave_struct guest_fx_image;
@@ -654,6 +672,8 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu);
int kvm_mmu_load(struct kvm_vcpu *vcpu);
void kvm_mmu_unload(struct kvm_vcpu *vcpu);
void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu);
+void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu);
+void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu);
int kvm_emulate_hypercall(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index a902479..31e837b 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -19,6 +19,7 @@
#include "mmu.h"
#include "kvm_cache_regs.h"
+#include "x86.h"
#include <linux/kvm_host.h>
#include <linux/types.h>
@@ -188,6 +189,7 @@ typedef int (*mmu_parent_walk_fn) (struct kvm_vcpu *vcpu, struct kvm_mmu_page *s
static struct kmem_cache *pte_chain_cache;
static struct kmem_cache *rmap_desc_cache;
static struct kmem_cache *mmu_page_header_cache;
+static struct kmem_cache *mmu_async_pf_cache;
static u64 __read_mostly shadow_trap_nonpresent_pte;
static u64 __read_mostly shadow_notrap_nonpresent_pte;
@@ -2189,6 +2191,218 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
error_code & PFERR_WRITE_MASK, gfn);
}
+static int gfn_to_pfn_async(struct kvm *kvm, gfn_t gfn, pfn_t *pfn)
+{
+ struct page *page[1];
+ unsigned long addr;
+ int npages;
+
+ *pfn = bad_pfn;
+
+ addr = gfn_to_hva(kvm, gfn);
+ if (kvm_is_error_hva(addr)) {
+ get_page(bad_page);
+ return 1;
+ }
+
+ npages = __get_user_pages_fast(addr, 1, 1, page);
+
+ if (unlikely(npages != 1)) {
+ down_read(¤t->mm->mmap_sem);
+ npages = get_user_pages_noio(current, current->mm, addr, 1, 1,
+ 0, page, NULL);
+ up_read(¤t->mm->mmap_sem);
+ }
+
+ if (unlikely(npages != 1)) {
+ struct vm_area_struct *vma;
+
+ down_read(¤t->mm->mmap_sem);
+ vma = find_vma(current->mm, addr);
+
+ if (vma == NULL || addr < vma->vm_start ||
+ !(vma->vm_flags & VM_PFNMAP)) {
+ up_read(¤t->mm->mmap_sem);
+ return 0; /* do async fault in */
+ }
+
+ *pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
+ up_read(¤t->mm->mmap_sem);
+ BUG_ON(!kvm_is_mmio_pfn(*pfn));
+ } else
+ *pfn = page_to_pfn(page[0]);
+
+ return 1;
+}
+
+static void async_pf_work_free(struct kvm_mmu_async_pf *apf)
+{
+ if (atomic_dec_and_test(&apf->used))
+ kmem_cache_free(mmu_async_pf_cache, apf);
+}
+
+static int async_pf_get_ref(struct slow_work *work)
+{
+ struct kvm_mmu_async_pf *apf =
+ container_of(work, struct kvm_mmu_async_pf, work);
+
+ atomic_inc(&apf->used);
+ return 0;
+}
+
+static void async_pf_put_ref(struct slow_work *work)
+{
+ struct kvm_mmu_async_pf *apf =
+ container_of(work, struct kvm_mmu_async_pf, work);
+
+ kvm_put_kvm(apf->vcpu->kvm);
+ async_pf_work_free(apf);
+}
+
+static void async_pf_execute(struct slow_work *work)
+{
+ struct page *page[1];
+ struct kvm_mmu_async_pf *apf =
+ container_of(work, struct kvm_mmu_async_pf, work);
+ wait_queue_head_t *q = &apf->vcpu->wq;
+
+ might_sleep();
+
+ down_read(&apf->mm->mmap_sem);
+ get_user_pages(current, apf->mm, apf->addr, 1, 1, 0, page, NULL);
+ up_read(&apf->mm->mmap_sem);
+
+ spin_lock(&apf->vcpu->arch.mmu_async_pf_lock);
+ list_add_tail(&apf->link, &apf->vcpu->arch.mmu_async_pf_done);
+ apf->page = page[0];
+ spin_unlock(&apf->vcpu->arch.mmu_async_pf_lock);
+
+ trace_kvm_mmu_async_pf_executed(apf->addr, apf->page, apf->token,
+ apf->gva);
+
+ if (waitqueue_active(q))
+ wake_up_interruptible(q);
+
+ mmdrop(apf->mm);
+}
+
+struct slow_work_ops async_pf_ops = {
+ .get_ref = async_pf_get_ref,
+ .put_ref = async_pf_put_ref,
+ .execute = async_pf_execute
+};
+
+void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu)
+{
+ while (!list_empty(&vcpu->arch.mmu_async_pf_done)) {
+ struct kvm_mmu_async_pf *work =
+ list_entry(vcpu->arch.mmu_async_pf_done.next,
+ typeof(*work), link);
+ list_del(&work->link);
+ put_page(work->page);
+ kmem_cache_free(mmu_async_pf_cache, work);
+ }
+}
+
+void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
+{
+ struct kvm_mmu_async_pf *work = vcpu->arch.mmu_async_pf_work;
+
+ if (work) {
+ vcpu->arch.mmu_async_pf_work = NULL;
+ if (work->page == NULL) {
+ vcpu->arch.pv_shm->reason = KVM_PV_REASON_PAGE_NP;
+ vcpu->arch.pv_shm->param = work->token;
+ kvm_inject_page_fault(vcpu, work->gva, 0);
+ trace_kvm_mmu_send_async_pf(
+ vcpu->arch.pv_shm->param,
+ work->gva, KVM_PV_REASON_PAGE_NP);
+ return;
+ } else {
+ spin_lock(&vcpu->arch.mmu_async_pf_lock);
+ list_del(&work->link);
+ spin_unlock(&vcpu->arch.mmu_async_pf_lock);
+ put_page(work->page);
+ async_pf_work_free(work);
+ }
+ }
+
+ if (list_empty_careful(&vcpu->arch.mmu_async_pf_done) ||
+ kvm_event_needs_reinjection(vcpu) ||
+ !kvm_x86_ops->interrupt_allowed(vcpu))
+ return;
+
+ spin_lock(&vcpu->arch.mmu_async_pf_lock);
+ work = list_first_entry(&vcpu->arch.mmu_async_pf_done, typeof(*work),
+ link);
+ list_del(&work->link);
+ spin_unlock(&vcpu->arch.mmu_async_pf_lock);
+
+ vcpu->arch.pv_shm->reason = KVM_PV_REASON_PAGE_READY;
+ vcpu->arch.pv_shm->param = work->token;
+ kvm_inject_page_fault(vcpu, work->gva, 0);
+ trace_kvm_mmu_send_async_pf(work->token, work->gva,
+ KVM_PV_REASON_PAGE_READY);
+
+ put_page(work->page);
+ async_pf_work_free(work);
+}
+
+static bool can_do_async_pf(struct kvm_vcpu *vcpu)
+{
+ struct kvm_segment kvm_seg;
+
+ if (!vcpu->arch.pv_shm ||
+ !(vcpu->arch.pv_shm->features & KVM_PV_SHM_FEATURES_ASYNC_PF) ||
+ kvm_event_needs_reinjection(vcpu))
+ return false;
+
+ kvm_get_segment(vcpu, &kvm_seg, VCPU_SREG_CS);
+
+ /* is userspace code? TODO check VM86 mode */
+ return !!(kvm_seg.selector & 3);
+}
+
+static int setup_async_pf(struct kvm_vcpu *vcpu, gva_t gva, gfn_t gfn)
+{
+ struct kvm_mmu_async_pf *work;
+
+ /* setup slow work */
+
+ /* do alloc atomic since if we are going to sleep anyway we
+ may as well sleep faulting in page */
+ work = kmem_cache_zalloc(mmu_async_pf_cache, GFP_ATOMIC);
+ if (!work)
+ return 0;
+
+ atomic_set(&work->used, 1);
+ work->page = NULL;
+ work->vcpu = vcpu;
+ work->gva = gva;
+ work->addr = gfn_to_hva(vcpu->kvm, gfn);
+ work->token = (vcpu->arch.async_pf_id++ << 12) | vcpu->vcpu_id;
+ work->mm = current->mm;
+ atomic_inc(&work->mm->mm_count);
+ kvm_get_kvm(work->vcpu->kvm);
+
+ /* this can't really happen otherwise gfn_to_pfn_async
+ would succeed */
+ if (unlikely(kvm_is_error_hva(work->addr)))
+ goto retry_sync;
+
+ slow_work_init(&work->work, &async_pf_ops);
+ if (slow_work_enqueue(&work->work) != 0)
+ goto retry_sync;
+
+ vcpu->arch.mmu_async_pf_work = work;
+ return 1;
+retry_sync:
+ kvm_put_kvm(work->vcpu->kvm);
+ mmdrop(work->mm);
+ kmem_cache_free(mmu_async_pf_cache, work);
+ return 0;
+}
+
static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
u32 error_code)
{
@@ -2211,7 +2425,23 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
mmu_seq = vcpu->kvm->mmu_notifier_seq;
smp_rmb();
- pfn = gfn_to_pfn(vcpu->kvm, gfn);
+
+ if (can_do_async_pf(vcpu)) {
+ r = gfn_to_pfn_async(vcpu->kvm, gfn, &pfn);
+ trace_kvm_mmu_try_async_get_page(r, pfn);
+ } else {
+do_sync:
+ r = 1;
+ pfn = gfn_to_pfn(vcpu->kvm, gfn);
+ }
+
+ if (!r) {
+ if (!setup_async_pf(vcpu, gpa, gfn))
+ goto do_sync;
+ return 0;
+ }
+
+ /* mmio */
if (is_error_pfn(pfn)) {
kvm_release_pfn_clean(pfn);
return 1;
@@ -2220,8 +2450,8 @@ static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
if (mmu_notifier_retry(vcpu, mmu_seq))
goto out_unlock;
kvm_mmu_free_some_pages(vcpu);
- r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
- level, gfn, pfn);
+ r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK, level, gfn,
+ pfn);
spin_unlock(&vcpu->kvm->mmu_lock);
return r;
@@ -2976,6 +3206,8 @@ static void mmu_destroy_caches(void)
kmem_cache_destroy(rmap_desc_cache);
if (mmu_page_header_cache)
kmem_cache_destroy(mmu_page_header_cache);
+ if (mmu_async_pf_cache)
+ kmem_cache_destroy(mmu_async_pf_cache);
}
void kvm_mmu_module_exit(void)
@@ -3003,6 +3235,11 @@ int kvm_mmu_module_init(void)
if (!mmu_page_header_cache)
goto nomem;
+ mmu_async_pf_cache = KMEM_CACHE(kvm_mmu_async_pf, 0);
+
+ if (!mmu_async_pf_cache)
+ goto nomem;
+
register_shrinker(&mmu_shrinker);
return 0;
diff --git a/arch/x86/kvm/mmutrace.h b/arch/x86/kvm/mmutrace.h
index 3e4a5c6..d6dd63c 100644
--- a/arch/x86/kvm/mmutrace.h
+++ b/arch/x86/kvm/mmutrace.h
@@ -214,6 +214,66 @@ TRACE_EVENT(
TP_printk("%s", KVM_MMU_PAGE_PRINTK())
);
+TRACE_EVENT(
+ kvm_mmu_try_async_get_page,
+ TP_PROTO(bool r, u64 pfn),
+ TP_ARGS(r, pfn),
+
+ TP_STRUCT__entry(
+ __field(__u64, pfn)
+ ),
+
+ TP_fast_assign(
+ __entry->pfn = r ? pfn : (u64)-1;
+ ),
+
+ TP_printk("pfn %#llx", __entry->pfn)
+);
+
+TRACE_EVENT(
+ kvm_mmu_send_async_pf,
+ TP_PROTO(u64 task, u64 gva, u64 reason),
+ TP_ARGS(task, gva, reason),
+
+ TP_STRUCT__entry(
+ __field(__u64, task)
+ __field(__u64, gva)
+ __field(bool, np)
+ ),
+
+ TP_fast_assign(
+ __entry->task = task;
+ __entry->gva = gva;
+ __entry->np = (reason == KVM_PV_REASON_PAGE_NP);
+ ),
+
+ TP_printk("task %#llx gva %#llx %s", __entry->task, __entry->gva,
+ __entry->np ? "not present" : "ready")
+);
+
+TRACE_EVENT(
+ kvm_mmu_async_pf_executed,
+ TP_PROTO(unsigned long address, struct page *page, u64 task, u64 gva),
+ TP_ARGS(address, page, task, gva),
+
+ TP_STRUCT__entry(
+ __field(unsigned long, address)
+ __field(struct page*, page)
+ __field(u64, task)
+ __field(u64, gva)
+ ),
+
+ TP_fast_assign(
+ __entry->address = address;
+ __entry->page = page;
+ __entry->task = task;
+ __entry->gva = gva;
+ ),
+
+ TP_printk("task %#llx gva %#llx address %#lx pfn %lx", __entry->task,
+ __entry->gva, __entry->address, page_to_pfn(__entry->page))
+);
+
#endif /* _TRACE_KVMMMU_H */
/* This part must be outside protection */
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 72558f8..9fe2ecd 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -419,7 +419,21 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
mmu_seq = vcpu->kvm->mmu_notifier_seq;
smp_rmb();
- pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
+
+ if (can_do_async_pf(vcpu)) {
+ r = gfn_to_pfn_async(vcpu->kvm, walker.gfn, &pfn);
+ trace_kvm_mmu_try_async_get_page(r, pfn);
+ } else {
+do_sync:
+ r = 1;
+ pfn = gfn_to_pfn(vcpu->kvm, walker.gfn);
+ }
+
+ if (!r) {
+ if (!setup_async_pf(vcpu, addr, walker.gfn))
+ goto do_sync;
+ return 0;
+ }
/* mmio */
if (is_error_pfn(pfn)) {
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c177933..e6bd3ad 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -3330,6 +3330,10 @@ int kvm_arch_init(void *opaque)
goto out;
}
+ r = slow_work_register_user();
+ if (r)
+ goto out;
+
r = kvm_mmu_module_init();
if (r)
goto out;
@@ -3352,6 +3356,7 @@ out:
void kvm_arch_exit(void)
{
+ slow_work_unregister_user();
if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
CPUFREQ_TRANSITION_NOTIFIER);
@@ -3837,6 +3842,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
}
}
+ kvm_check_async_pf_completion(vcpu);
+
preempt_disable();
kvm_x86_ops->prepare_guest_switch(vcpu);
@@ -3927,7 +3934,6 @@ out:
return r;
}
-
static int __vcpu_run(struct kvm_vcpu *vcpu)
{
int r;
@@ -5026,6 +5032,9 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
}
vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS;
+ INIT_LIST_HEAD(&vcpu->arch.mmu_async_pf_done);
+ spin_lock_init(&vcpu->arch.mmu_async_pf_lock);
+
return 0;
fail_mmu_destroy:
@@ -5078,8 +5087,10 @@ static void kvm_free_vcpus(struct kvm *kvm)
/*
* Unpin any mmu pages first.
*/
- kvm_for_each_vcpu(i, vcpu, kvm)
+ kvm_for_each_vcpu(i, vcpu, kvm) {
+ kvm_clear_async_pf_completion_queue(vcpu);
kvm_unload_vcpu_mmu(vcpu);
+ }
kvm_for_each_vcpu(i, vcpu, kvm)
kvm_arch_vcpu_free(vcpu);
@@ -5178,10 +5189,11 @@ void kvm_arch_flush_shadow(struct kvm *kvm)
int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
{
return vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE
+ || !list_empty_careful(&vcpu->arch.mmu_async_pf_done)
|| vcpu->arch.mp_state == KVM_MP_STATE_SIPI_RECEIVED
- || vcpu->arch.nmi_pending ||
- (kvm_arch_interrupt_allowed(vcpu) &&
- kvm_cpu_has_interrupt(vcpu));
+ || vcpu->arch.nmi_pending
+ || (kvm_arch_interrupt_allowed(vcpu) &&
+ kvm_cpu_has_interrupt(vcpu));
}
void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
--
1.6.3.3
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
