perf report shows heavy overhead from down/up of slots_lock.
Attempted to remove slots_lock by having vcpus stop on a synchronization
point, but this introduced further complexity (a vcpu can be scheduled
out before reaching the synchronization point, and can sched back in at
points which are slots_lock protected, etc).
This patch changes vcpu_enter_guest to conditionally release/acquire
slots_lock in case a vcpu state bit is set.
vmexit performance improves by 5-10% on UP guest.
Signed-off-by: Marcelo Tosatti <mtosatti@xxxxxxxxxx>
Index: kvm-requests/arch/x86/kvm/vmx.c
===================================================================
--- kvm-requests.orig/arch/x86/kvm/vmx.c
+++ kvm-requests/arch/x86/kvm/vmx.c
@@ -2169,7 +2169,7 @@ static int alloc_apic_access_page(struct
struct kvm_userspace_memory_region kvm_userspace_mem;
int r = 0;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
if (kvm->arch.apic_access_page)
goto out;
kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT;
@@ -2191,7 +2191,7 @@ static int alloc_identity_pagetable(stru
struct kvm_userspace_memory_region kvm_userspace_mem;
int r = 0;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
if (kvm->arch.ept_identity_pagetable)
goto out;
kvm_userspace_mem.slot = IDENTITY_PAGETABLE_PRIVATE_MEMSLOT;
Index: kvm-requests/arch/x86/kvm/x86.c
===================================================================
--- kvm-requests.orig/arch/x86/kvm/x86.c
+++ kvm-requests/arch/x86/kvm/x86.c
@@ -1926,7 +1926,7 @@ static int kvm_vm_ioctl_set_nr_mmu_pages
if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES)
return -EINVAL;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
spin_lock(&kvm->mmu_lock);
kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages);
@@ -1982,7 +1982,7 @@ static int kvm_vm_ioctl_set_memory_alias
< alias->target_phys_addr)
goto out;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
spin_lock(&kvm->mmu_lock);
p = &kvm->arch.aliases[alias->slot];
@@ -2137,7 +2137,7 @@ int kvm_vm_ioctl_get_dirty_log(struct kv
struct kvm_memory_slot *memslot;
int is_dirty = 0;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
r = kvm_get_dirty_log(kvm, log, &is_dirty);
if (r)
@@ -2253,7 +2253,7 @@ long kvm_arch_vm_ioctl(struct file *filp
sizeof(struct kvm_pit_config)))
goto out;
create_pit:
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
r = -EEXIST;
if (kvm->arch.vpit)
goto create_pit_unlock;
@@ -3548,7 +3548,7 @@ static void inject_pending_event(struct
static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
{
- int r;
+ int r, dropped_slots_lock = 0;
bool req_int_win = !irqchip_in_kernel(vcpu->kvm) &&
vcpu->run->request_interrupt_window;
@@ -3616,7 +3616,10 @@ static int vcpu_enter_guest(struct kvm_v
kvm_lapic_sync_to_vapic(vcpu);
}
- up_read(&vcpu->kvm->slots_lock);
+ if (unlikely(test_bit(KVM_VCPU_DROP_LOCK, &vcpu->vcpu_state))) {
+ dropped_slots_lock = 1;
+ up_read(&vcpu->kvm->slots_lock);
+ }
kvm_guest_enter();
@@ -3668,8 +3671,8 @@ static int vcpu_enter_guest(struct kvm_v
preempt_enable();
- down_read(&vcpu->kvm->slots_lock);
-
+ if (dropped_slots_lock)
+ down_read(&vcpu->kvm->slots_lock);
/*
* Profile KVM exit RIPs:
*/
Index: kvm-requests/include/linux/kvm_host.h
===================================================================
--- kvm-requests.orig/include/linux/kvm_host.h
+++ kvm-requests/include/linux/kvm_host.h
@@ -44,6 +44,7 @@
#define KVM_VCPU_GUEST_MODE 0
#define KVM_VCPU_KICKED 1
+#define KVM_VCPU_DROP_LOCK 2
struct kvm;
struct kvm_vcpu;
@@ -408,6 +409,7 @@ void kvm_unregister_irq_ack_notifier(str
struct kvm_irq_ack_notifier *kian);
int kvm_request_irq_source_id(struct kvm *kvm);
void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id);
+void kvm_grab_global_lock(struct kvm *kvm);
/* For vcpu->arch.iommu_flags */
#define KVM_IOMMU_CACHE_COHERENCY 0x1
Index: kvm-requests/virt/kvm/coalesced_mmio.c
===================================================================
--- kvm-requests.orig/virt/kvm/coalesced_mmio.c
+++ kvm-requests/virt/kvm/coalesced_mmio.c
@@ -117,7 +117,7 @@ int kvm_vm_ioctl_register_coalesced_mmio
if (dev == NULL)
return -EINVAL;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
if (dev->nb_zones >= KVM_COALESCED_MMIO_ZONE_MAX) {
up_write(&kvm->slots_lock);
return -ENOBUFS;
@@ -140,7 +140,7 @@ int kvm_vm_ioctl_unregister_coalesced_mm
if (dev == NULL)
return -EINVAL;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
i = dev->nb_zones;
while(i) {
Index: kvm-requests/virt/kvm/eventfd.c
===================================================================
--- kvm-requests.orig/virt/kvm/eventfd.c
+++ kvm-requests/virt/kvm/eventfd.c
@@ -498,7 +498,7 @@ kvm_assign_ioeventfd(struct kvm *kvm, st
else
p->wildcard = true;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
/* Verify that there isnt a match already */
if (ioeventfd_check_collision(kvm, p)) {
@@ -541,7 +541,7 @@ kvm_deassign_ioeventfd(struct kvm *kvm,
if (IS_ERR(eventfd))
return PTR_ERR(eventfd);
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) {
bool wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH);
Index: kvm-requests/virt/kvm/kvm_main.c
===================================================================
--- kvm-requests.orig/virt/kvm/kvm_main.c
+++ kvm-requests/virt/kvm/kvm_main.c
@@ -787,6 +787,22 @@ void kvm_reload_remote_mmus(struct kvm *
kvm_vcpus_request(kvm, KVM_REQ_MMU_RELOAD);
}
+void kvm_grab_global_lock(struct kvm *kvm)
+{
+ int i;
+ struct kvm_vcpu *vcpu;
+
+ kvm_for_each_vcpu(i, vcpu, kvm) {
+ set_bit(KVM_VCPU_DROP_LOCK, &vcpu->vcpu_state);
+ barrier();
+ kvm_vcpu_ipi(vcpu);
+ }
+ down_write(&kvm->slots_lock);
+ kvm_for_each_vcpu(i, vcpu, kvm)
+ clear_bit(KVM_VCPU_DROP_LOCK, &vcpu->vcpu_state);
+}
+EXPORT_SYMBOL_GPL(kvm_grab_global_lock);
+
int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
struct page *page;
@@ -1286,7 +1302,7 @@ int kvm_set_memory_region(struct kvm *kv
{
int r;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
r = __kvm_set_memory_region(kvm, mem, user_alloc);
up_write(&kvm->slots_lock);
return r;
@@ -2556,7 +2572,7 @@ int kvm_io_bus_register_dev(struct kvm *
{
int ret;
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
ret = __kvm_io_bus_register_dev(bus, dev);
up_write(&kvm->slots_lock);
@@ -2579,7 +2595,7 @@ void kvm_io_bus_unregister_dev(struct kv
struct kvm_io_bus *bus,
struct kvm_io_device *dev)
{
- down_write(&kvm->slots_lock);
+ kvm_grab_global_lock(kvm);
__kvm_io_bus_unregister_dev(bus, dev);
up_write(&kvm->slots_lock);
}
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
