This makes it possible to batch notifications, and the fewer barriers
helps performance:
Before: (./vringh_test --eventidx --parallel):
Using CPUS 0 and 3
Guest: notified 156428, pinged 156251
Host: notified 156251, pinged 78215
R=4.575 U=3.520 S=3.520
Using CPUS 0 and 3
Guest: notified 156617, pinged 156250
Host: notified 156250, pinged 78310
R=4.511 U=3.580 S=3.580
Using CPUS 0 and 3
Guest: notified 156357, pinged 156251
Host: notified 156251, pinged 78179
R=4.518 U=3.536 S=3.536
After:
Using CPUS 0 and 3
Guest: notified 156394, pinged 156251
Host: notified 156251, pinged 78197
R=4.282 U=3.456 S=3.456
Using CPUS 0 and 3
Guest: notified 156578, pinged 156251
Host: notified 156251, pinged 78289
R=4.248 U=3.436 S=3.436
Using CPUS 0 and 3
Guest: notified 156594, pinged 156251
Host: notified 156251, pinged 78297
R=4.329 U=3.416 S=3.416
Signed-off-by: Rusty Russell <rusty@xxxxxxxxxxxxxxx>
---
drivers/vhost/vringh.c | 89 +++++++++++++++++++++++++++++++++---------------
include/linux/vringh.h | 15 +++++---
2 files changed, 72 insertions(+), 32 deletions(-)
diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c
index b28670f..ab10da8 100644
--- a/drivers/vhost/vringh.c
+++ b/drivers/vhost/vringh.c
@@ -290,13 +290,12 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len,
int (*putu16)(u16 *p, u16 val),
int (*putused)(struct vring_used_elem *dst,
const struct vring_used_elem
- *s),
- bool *notify)
+ *s))
{
struct vring_used_elem used;
struct vring_used *used_ring;
int err;
- u16 used_idx, old, used_event;
+ u16 used_idx;
used.id = idx;
used.len = len;
@@ -309,7 +308,7 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len,
}
used_ring = vrh->vring.used;
- used_idx = vrh->last_used_idx;
+ used_idx = vrh->last_used_idx + vrh->completed;
err = putused(&used_ring->ring[used_idx % vrh->vring.num],
&used);
@@ -323,19 +322,24 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len,
/* Make sure buffer is written before we update index. */
virtio_wmb(vrh->weak_barriers);
- old = vrh->last_used_idx;
- vrh->last_used_idx++;
-
- err = putu16(&vrh->vring.used->idx, vrh->last_used_idx);
+ err = putu16(&vrh->vring.used->idx, used_idx + 1);
if (err) {
vringh_bad("Failed to update used index at %p",
&vrh->vring.used->idx);
return err;
}
- /* If we already know we need to notify, skip re-checking */
- if (*notify)
- return 0;
+ vrh->completed++;
+ return 0;
+}
+
+
+static inline int __vringh_need_notify(struct vringh *vrh,
+ int (*getu16)(u16 *val, const u16 *p))
+{
+ bool notify;
+ u16 used_event;
+ int err;
/* Flush out used index update. This is paired with the
* barrier that the Guest executes when enabling
@@ -351,21 +355,28 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len,
&vrh->vring.avail->flags);
return err;
}
- if (!(flags & VRING_AVAIL_F_NO_INTERRUPT))
- *notify = true;
- return 0;
+ return (!(flags & VRING_AVAIL_F_NO_INTERRUPT));
}
- /* Modern: we know where other side is up to. */
+ /* Modern: we know when other side wants to know. */
err = getu16(&used_event, &vring_used_event(&vrh->vring));
if (err) {
vringh_bad("Failed to get used event idx at %p",
&vring_used_event(&vrh->vring));
return err;
}
- if (vring_need_event(used_event, vrh->last_used_idx, old))
- *notify = true;
- return 0;
+
+ /* Just in case we added so many that we wrap. */
+ if (unlikely(vrh->completed > 0xffff))
+ notify = true;
+ else
+ notify = vring_need_event(used_event,
+ vrh->last_used_idx + vrh->completed,
+ vrh->last_used_idx);
+
+ vrh->last_used_idx += vrh->completed;
+ vrh->completed = 0;
+ return notify;
}
static inline bool __vringh_notify_enable(struct vringh *vrh,
@@ -587,14 +598,14 @@ void vringh_abandon_user(struct vringh *vrh, unsigned int num)
* @vrh: the vring.
* @head: the head as filled in by vringh_getdesc_user.
* @len: the length of data we have written.
- * @notify: set if we should notify the other side, otherwise left alone.
+ *
+ * You should check vringh_need_notify_user() after one or more calls
+ * to this function.
*/
-int vringh_complete_user(struct vringh *vrh, u16 head, u32 len,
- bool *notify)
+int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)
{
return __vringh_complete(vrh, head, len,
- getu16_user, putu16_user, putused_user,
- notify);
+ getu16_user, putu16_user, putused_user);
}
/**
@@ -621,6 +632,17 @@ void vringh_notify_disable_user(struct vringh *vrh)
__vringh_notify_disable(vrh, putu16_user);
}
+/**
+ * vringh_need_notify_user - must we tell the other side about used buffers?
+ * @vrh: the vring we've called vringh_complete_user() on.
+ *
+ * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
+ */
+int vringh_need_notify_user(struct vringh *vrh)
+{
+ return __vringh_need_notify(vrh, getu16_user);
+}
+
/* Kernelspace access helpers. */
static inline int getu16_kern(u16 *val, const u16 *p)
{
@@ -783,14 +805,14 @@ void vringh_abandon_kern(struct vringh *vrh, unsigned int num)
* @vrh: the vring.
* @head: the head as filled in by vringh_getdesc_kern.
* @len: the length of data we have written.
- * @notify: set if we should notify the other side, otherwise left alone.
+ *
+ * You should check vringh_need_notify_kern() after one or more calls
+ * to this function.
*/
-int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len,
- bool *notify)
+int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)
{
return __vringh_complete(vrh, head, len,
- getu16_kern, putu16_kern, putused_kern,
- notify);
+ getu16_kern, putu16_kern, putused_kern);
}
/**
@@ -816,3 +838,14 @@ void vringh_notify_disable_kern(struct vringh *vrh)
{
__vringh_notify_disable(vrh, putu16_kern);
}
+
+/**
+ * vringh_need_notify_kern - must we tell the other side about used buffers?
+ * @vrh: the vring we've called vringh_complete_kern() on.
+ *
+ * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
+ */
+int vringh_need_notify_kern(struct vringh *vrh)
+{
+ return __vringh_need_notify(vrh, getu16_kern);
+}
diff --git a/include/linux/vringh.h b/include/linux/vringh.h
index 508b5e5..9df86e9 100644
--- a/include/linux/vringh.h
+++ b/include/linux/vringh.h
@@ -44,6 +44,9 @@ struct vringh {
/* Last index we used. */
u16 last_used_idx;
+ /* How many descriptors we've completed since last need_notify(). */
+ u32 completed;
+
/* The vring (note: it may contain user pointers!) */
struct vring vring;
};
@@ -83,13 +86,15 @@ ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len);
ssize_t vringh_iov_push_user(struct vringh_iov *wiov,
const void *src, size_t len);
-/* Mark a descriptor as used. Sets notify if you should fire eventfd. */
-int vringh_complete_user(struct vringh *vrh, u16 head, u32 len,
- bool *notify);
+/* Mark a descriptor as used. */
+int vringh_complete_user(struct vringh *vrh, u16 head, u32 len);
/* Pretend we've never seen descriptor (for easy error handling). */
void vringh_abandon_user(struct vringh *vrh, unsigned int num);
+/* Do we need to fire the eventfd to notify the other side? */
+int vringh_need_notify_user(struct vringh *vrh);
+
/* Helpers for kernelspace vrings. */
int vringh_init_kern(struct vringh *vrh, u32 features,
unsigned int num, bool weak_barriers,
@@ -107,9 +112,11 @@ ssize_t vringh_iov_pull_kern(struct vringh_iov *riov, void *dst, size_t len);
ssize_t vringh_iov_push_user(struct vringh_iov *wiov,
const void *src, size_t len);
void vringh_abandon_kern(struct vringh *vrh, unsigned int num);
-int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len, bool *notify);
+int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len);
bool vringh_notify_enable_kern(struct vringh *vrh);
void vringh_notify_disable_kern(struct vringh *vrh);
+int vringh_need_notify_kern(struct vringh *vrh);
+
#endif /* _LINUX_VRINGH_H */
--
1.7.10.4
_______________________________________________
Virtualization mailing list
Virtualization@xxxxxxxxxxxxxxxxxxxxxxxxxx
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
