"rep ins" emulation is going through emulator now. This is slow because
emulator knows how to write back only one datum at a time. This patch
provides fast path for the instruction in certain conditions. The
conditions are: DF flag is not set, destination memory is RAM and single
datum does not cross page boundary. If fast path code fails it falls
back to emulation.
Signed-off-by: Gleb Natapov <gleb@xxxxxxxxxx>
---
arch/x86/include/asm/kvm_host.h | 5 ++
arch/x86/kvm/svm.c | 20 ++++++--
arch/x86/kvm/vmx.c | 25 +++++++---
arch/x86/kvm/x86.c | 95 +++++++++++++++++++++++++++++++++++++++
4 files changed, 133 insertions(+), 12 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 1bcd280..5129639 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -403,6 +403,9 @@ struct kvm_vcpu_arch {
/* emulate context */
struct x86_emulate_ctxt emulate_ctxt;
+ struct x86_fast_string_pio_ctxt {
+ unsigned long linear_addr;
+ } fast_string_pio_ctxt;
bool emulate_regs_need_sync_to_vcpu;
bool emulate_regs_need_sync_from_vcpu;
int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
@@ -763,6 +766,8 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data);
struct x86_emulate_ctxt;
int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port);
+int kvm_fast_string_pio_in(struct kvm_vcpu *vcpu, int size, unsigned short port,
+ u8 addr_size);
void kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
int kvm_emulate_halt(struct kvm_vcpu *vcpu);
int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index f75af40..20e3fb0 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -1887,21 +1887,31 @@ static int io_interception(struct vcpu_svm *svm)
{
struct kvm_vcpu *vcpu = &svm->vcpu;
u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
- int size, in, string;
+ int size, in, string, rep;
unsigned port;
++svm->vcpu.stat.io_exits;
string = (io_info & SVM_IOIO_STR_MASK) != 0;
+ rep = (io_info & SVM_IOIO_REP_MASK) != 0;
in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
- if (string || in)
- return emulate_instruction(vcpu, 0) == EMULATE_DONE;
port = io_info >> 16;
size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
svm->next_rip = svm->vmcb->control.exit_info_2;
- skip_emulated_instruction(&svm->vcpu);
- return kvm_fast_pio_out(vcpu, size, port);
+ if (!string && !in) {
+ skip_emulated_instruction(&svm->vcpu);
+ return kvm_fast_pio_out(vcpu, size, port);
+ } else if (string && in && rep) {
+ int addr_size = (io_info & SVM_IOIO_ASIZE_MASK) >>
+ SVM_IOIO_ASIZE_SHIFT;
+ int r = kvm_fast_string_pio_in(vcpu, size, port,
+ ffs(addr_size) - 1);
+ if (r != EMULATE_FAIL)
+ return r == EMULATE_DONE;
+ }
+
+ return emulate_instruction(vcpu, 0) == EMULATE_DONE;
}
static int nmi_interception(struct vcpu_svm *svm)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 32eb588..a2935f3 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -632,6 +632,7 @@ static unsigned long *vmx_msr_bitmap_longmode;
static bool cpu_has_load_ia32_efer;
static bool cpu_has_load_perf_global_ctrl;
+static bool cpu_has_ins_outs_inst_info;
static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
static DEFINE_SPINLOCK(vmx_vpid_lock);
@@ -2510,6 +2511,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
if (((vmx_msr_high >> 18) & 15) != 6)
return -EIO;
+ cpu_has_ins_outs_inst_info = vmx_msr_high & (1u << 22);
+
vmcs_conf->size = vmx_msr_high & 0x1fff;
vmcs_conf->order = get_order(vmcs_config.size);
vmcs_conf->revision_id = vmx_msr_low;
@@ -4333,23 +4336,31 @@ static int handle_triple_fault(struct kvm_vcpu *vcpu)
static int handle_io(struct kvm_vcpu *vcpu)
{
unsigned long exit_qualification;
- int size, in, string;
+ int size, in, string, rep;
unsigned port;
exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
- string = (exit_qualification & 16) != 0;
in = (exit_qualification & 8) != 0;
+ string = (exit_qualification & 16) != 0;
+ rep = (exit_qualification & 32) != 0;
++vcpu->stat.io_exits;
- if (string || in)
- return emulate_instruction(vcpu, 0) == EMULATE_DONE;
-
port = exit_qualification >> 16;
size = (exit_qualification & 7) + 1;
- skip_emulated_instruction(vcpu);
- return kvm_fast_pio_out(vcpu, size, port);
+ if (!string && !in) {
+ skip_emulated_instruction(vcpu);
+ return kvm_fast_pio_out(vcpu, size, port);
+ } else if (string && in && rep && cpu_has_ins_outs_inst_info) {
+ u32 inst_info = vmcs_read32(VMX_INSTRUCTION_INFO);
+ int r = kvm_fast_string_pio_in(vcpu, size, port,
+ (inst_info >> 7) & 7);
+ if (r != EMULATE_FAIL)
+ return r == EMULATE_DONE;
+ }
+
+ return emulate_instruction(vcpu, 0) == EMULATE_DONE;
}
static void
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 8361649..3bc7ad3 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4660,6 +4660,101 @@ int kvm_fast_pio_out(struct kvm_vcpu *vcpu, int size, unsigned short port)
}
EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
+static int __kvm_fast_string_pio_in(struct kvm_vcpu *vcpu, int size,
+ unsigned short port, unsigned long addr,
+ int count)
+{
+ struct page *page;
+ gpa_t gpa;
+ char *kaddr;
+ int ret;
+
+ gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);
+
+ if (gpa == UNMAPPED_GVA ||
+ (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
+ return EMULATE_FAIL;
+
+ page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
+ if (is_error_page(page)) {
+ kvm_release_page_clean(page);
+ return EMULATE_FAIL;
+ }
+
+ kaddr = kmap_atomic(page);
+ kaddr += offset_in_page(gpa);
+
+ ret = emulator_pio_in_emulated(&vcpu->arch.emulate_ctxt, size, port,
+ kaddr, count);
+
+ kunmap_atomic(kaddr);
+ if (ret) {
+ kvm_register_write(vcpu, VCPU_REGS_RCX,
+ kvm_register_read(vcpu, VCPU_REGS_RCX) - count);
+ kvm_register_write(vcpu, VCPU_REGS_RDI,
+ kvm_register_read(vcpu, VCPU_REGS_RDI) + count*size);
+ kvm_release_page_dirty(page);
+ return EMULATE_DONE;
+ }
+ kvm_release_page_clean(page);
+ return EMULATE_DO_MMIO;
+}
+
+static int complete_fast_string_pio(struct kvm_vcpu *vcpu)
+{
+ unsigned long linear_addr = vcpu->arch.fast_string_pio_ctxt.linear_addr;
+ int r;
+
+ BUG_ON(!vcpu->arch.pio.count);
+
+ vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
+ r = __kvm_fast_string_pio_in(vcpu, vcpu->arch.pio.size,
+ vcpu->arch.pio.port, linear_addr, vcpu->arch.pio.count);
+ BUG_ON(r == EMULATE_DO_MMIO);
+ if (r == EMULATE_FAIL) /* mem slot gone while we were not looking */
+ vcpu->arch.pio.count = 0; /* drop the pio data */
+ srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
+ return 1;
+}
+
+int kvm_fast_string_pio_in(struct kvm_vcpu *vcpu, int size,
+ unsigned short port, u8 addr_size)
+{
+ unsigned long masks[] = {0xffff, 0xffffffff, ~0};
+ unsigned long rdi = kvm_register_read(vcpu, VCPU_REGS_RDI);
+ unsigned long linear_addr = rdi + get_segment_base(vcpu, VCPU_SREG_ES);
+ unsigned long rcx = kvm_register_read(vcpu, VCPU_REGS_RCX), count;
+ int r;
+
+ if (rcx == 0) {
+ kvm_x86_ops->skip_emulated_instruction(vcpu);
+ return EMULATE_DONE;
+ }
+ if (kvm_get_rflags(vcpu) & X86_EFLAGS_DF)
+ return EMULATE_FAIL;
+ if (addr_size > 2)
+ return EMULATE_FAIL;
+
+ linear_addr &= masks[addr_size];
+
+ count = (PAGE_SIZE - offset_in_page(linear_addr))/size;
+
+ if (count == 0) /* 'in' crosses page boundry */
+ return EMULATE_FAIL;
+
+ count = min(count, rcx);
+
+ r = __kvm_fast_string_pio_in(vcpu, size, port, linear_addr, count);
+
+ if (r != EMULATE_DO_MMIO)
+ return r;
+
+ vcpu->arch.fast_string_pio_ctxt.linear_addr = linear_addr;
+ vcpu->arch.complete_userspace_io = complete_fast_string_pio;
+ return EMULATE_DO_MMIO;
+}
+EXPORT_SYMBOL_GPL(kvm_fast_string_pio_in);
+
static void tsc_bad(void *info)
{
__this_cpu_write(cpu_tsc_khz, 0);
--
1.7.7.3
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
