Wire up using an io_batch for f_op->iopoll(). If the lower stack supports
it, we can handle high rates of polled IO more efficiently.
This raises the single core efficiency on my system from ~6.1M IOPS to
~6.6M IOPS running a random read workload at depth 128 on two gen2
Optane drives.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
fs/io_uring.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 082ff64c1bcb..cbf00ad3ac3f 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2390,6 +2390,8 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
{
struct io_wq_work_node *pos, *start, *prev;
unsigned int poll_flags = BLK_POLL_NOSLEEP;
+ struct file *file = NULL;
+ struct io_batch ib;
int nr_events = 0;
/*
@@ -2399,11 +2401,17 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
if (ctx->poll_multi_queue && force_nonspin)
poll_flags |= BLK_POLL_ONESHOT;
+ ib.req_list = NULL;
wq_list_for_each(pos, start, &ctx->iopoll_list) {
struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);
struct kiocb *kiocb = &req->rw.kiocb;
int ret;
+ if (!file)
+ file = kiocb->ki_filp;
+ else if (file != kiocb->ki_filp)
+ break;
+
/*
* Move completed and retryable entries to our local lists.
* If we find a request that requires polling, break out
@@ -2412,19 +2420,21 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
if (READ_ONCE(req->iopoll_completed))
break;
- ret = kiocb->ki_filp->f_op->iopoll(kiocb, NULL, poll_flags);
+ ret = kiocb->ki_filp->f_op->iopoll(kiocb, &ib, poll_flags);
if (unlikely(ret < 0))
return ret;
else if (ret)
poll_flags |= BLK_POLL_ONESHOT;
/* iopoll may have completed current req */
- if (READ_ONCE(req->iopoll_completed))
+ if (ib.req_list || READ_ONCE(req->iopoll_completed))
break;
}
- if (!pos)
+ if (!pos && !ib.req_list)
return 0;
+ if (ib.req_list)
+ ib.complete(&ib);
prev = start;
wq_list_for_each_resume(pos, prev) {
--
2.33.0
