This patch is intended to release the CPU resources of io_uring in
polling mode. When IO is issued, the program immediately polls for
check completion, which is a waste of CPU resources when IO commands
are executed on the disk.
I add the hybrid polling feature in io_uring, enables polling to
release a portion of CPU resources without affecting block layer.
The CPU optimization in peak workload of patch is tested as follows:
set 8 poll queues
all CPU utilization of original polling is 100% for per CPU, after
optimization, the CPU utilization drop a lot (per CPU);
read(128k, QD64, 1Job) 37% write(128k, QD64, 1Job) 40%
randread(4k, QD64, 16Job) 52% randwrite(4k, QD64, 16Job) 12%
Compared to original polling, the optimised performance reduction
with peak workload within 1%.
read 0.29% write 0.51% randread 0.09% randwrite 0%
Signed-off-by: hexue <xue01.he@xxxxxxxxxxx>
---
changes:
v3:
- Simplified the patch comments
v2:
- extend hybrid poll to async polled io
v1:
- initial version
---
include/linux/io_uring_types.h | 14 ++++
include/uapi/linux/io_uring.h | 1 +
io_uring/io_uring.c | 4 +-
io_uring/io_uring.h | 3 +
io_uring/rw.c | 115 ++++++++++++++++++++++++++++++++-
5 files changed, 135 insertions(+), 2 deletions(-)
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 854ad67a5f70..3a75b9904326 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -224,6 +224,11 @@ struct io_alloc_cache {
size_t elem_size;
};
+struct iopoll_info {
+ long last_runtime;
+ long last_irqtime;
+};
+
struct io_ring_ctx {
/* const or read-mostly hot data */
struct {
@@ -421,6 +426,7 @@ struct io_ring_ctx {
unsigned short n_sqe_pages;
struct page **ring_pages;
struct page **sqe_pages;
+ struct xarray poll_array;
};
struct io_tw_state {
@@ -571,6 +577,12 @@ static inline void io_kiocb_cmd_sz_check(size_t cmd_sz)
)
#define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr)
+struct hy_poll_time {
+ int poll_state;
+ struct timespec64 iopoll_start;
+ struct timespec64 iopoll_end;
+};
+
struct io_kiocb {
union {
/*
@@ -641,6 +653,8 @@ struct io_kiocb {
u64 extra1;
u64 extra2;
} big_cqe;
+ /* for hybrid iopoll */
+ struct hy_poll_time *hy_poll;
};
struct io_overflow_cqe {
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 7a673b52827b..0038cdfec18f 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -198,6 +198,7 @@ enum {
* Removes indirection through the SQ index array.
*/
#define IORING_SETUP_NO_SQARRAY (1U << 16)
+#define IORING_SETUP_HY_POLL (1U << 17)
enum io_uring_op {
IORING_OP_NOP,
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index cd9a137ad6ce..2c14768bbe27 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -311,6 +311,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
goto err;
ctx->flags = p->flags;
+ xa_init(&ctx->poll_array);
atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
init_waitqueue_head(&ctx->sqo_sq_wait);
INIT_LIST_HEAD(&ctx->sqd_list);
@@ -2921,6 +2922,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
kfree(ctx->cancel_table_locked.hbs);
kfree(ctx->io_bl);
xa_destroy(&ctx->io_bl_xa);
+ xa_destroy(&ctx->poll_array);
kfree(ctx);
}
@@ -4050,7 +4052,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |
IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN |
IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY |
- IORING_SETUP_NO_SQARRAY))
+ IORING_SETUP_NO_SQARRAY | IORING_SETUP_HY_POLL))
return -EINVAL;
return io_uring_create(entries, &p, params);
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index d5495710c178..72d6a4c3b46d 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -125,6 +125,9 @@ static inline void io_req_task_work_add(struct io_kiocb *req)
__io_req_task_work_add(req, 0);
}
+/* if sleep time less than 1us, then do not do the schedule op */
+#define MIN_SCHETIME 1000
+
#define io_for_each_link(pos, head) \
for (pos = (head); pos; pos = pos->link)
diff --git a/io_uring/rw.c b/io_uring/rw.c
index d5e79d9bdc71..29c7ce23ed71 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -713,6 +713,46 @@ static bool need_complete_io(struct io_kiocb *req)
S_ISBLK(file_inode(req->file)->i_mode);
}
+void init_hybrid_poll(struct io_ring_ctx *ctx, struct io_kiocb *req)
+{
+ /*
+ * In multiple concurrency, a thread may operate several files
+ * under different file systems, the inode numbers may be
+ * duplicated. Each device has a different IO command processing
+ * capability, so using device number to record the running time
+ * of device
+ */
+ u32 index = req->file->f_inode->i_rdev;
+ struct iopoll_info *entry = xa_load(&ctx->poll_array, index);
+ struct hy_poll_time *hpt = kmalloc(sizeof(struct hy_poll_time), GFP_KERNEL);
+
+ /* if alloc fail, go to regular poll */
+ if (!hpt) {
+ ctx->flags &= ~IORING_SETUP_HY_POLL;
+ return;
+ }
+ hpt->poll_state = 0;
+ req->hy_poll = hpt;
+
+ if (!entry) {
+ entry = kmalloc(sizeof(struct iopoll_info), GFP_KERNEL);
+ if (!entry) {
+ ctx->flags &= ~IORING_SETUP_HY_POLL;
+ return;
+ }
+ entry->last_runtime = 0;
+ entry->last_irqtime = 0;
+ xa_store(&ctx->poll_array, index, entry, GFP_KERNEL);
+ }
+
+ /*
+ * Here we need nanosecond timestamps, some ways of reading
+ * timestamps directly are only accurate to microseconds, so
+ * there's no better alternative here for now
+ */
+ ktime_get_ts64(&hpt->iopoll_start);
+}
+
static int io_rw_init_file(struct io_kiocb *req, fmode_t mode)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@@ -750,6 +790,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode)
kiocb->ki_flags |= IOCB_HIPRI;
kiocb->ki_complete = io_complete_rw_iopoll;
req->iopoll_completed = 0;
+ if (ctx->flags & IORING_SETUP_HY_POLL)
+ init_hybrid_poll(ctx, req);
} else {
if (kiocb->ki_flags & IOCB_HIPRI)
return -EINVAL;
@@ -1118,6 +1160,75 @@ void io_rw_fail(struct io_kiocb *req)
io_req_set_res(req, res, req->cqe.flags);
}
+void io_delay(struct hy_poll_time *hpt, struct iopoll_info *entry)
+{
+ struct hrtimer_sleeper timer;
+ struct timespec64 tc, oldtc;
+ enum hrtimer_mode mode;
+ ktime_t kt;
+ long sleep_ti;
+
+ if (hpt->poll_state == 1)
+ return;
+
+ if (entry->last_runtime <= entry->last_irqtime)
+ return;
+
+ /*
+ * Avoid excessive scheduling time affecting performance
+ * by using only 25 per cent of the remaining time
+ */
+ sleep_ti = (entry->last_runtime - entry->last_irqtime) / 4;
+
+ /*
+ * If the time available for sleep is too short, i.e. the
+ * totle running time and the context switching loss time
+ * are very close to each other, the scheduling operation
+ * is not performed to avoid increasing latency
+ */
+ if (sleep_ti < MIN_SCHETIME)
+ return;
+
+ ktime_get_ts64(&oldtc);
+ kt = ktime_set(0, sleep_ti);
+ hpt->poll_state = 1;
+
+ mode = HRTIMER_MODE_REL;
+ hrtimer_init_sleeper_on_stack(&timer, CLOCK_MONOTONIC, mode);
+ hrtimer_set_expires(&timer.timer, kt);
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ hrtimer_sleeper_start_expires(&timer, mode);
+
+ if (timer.task)
+ io_schedule();
+
+ hrtimer_cancel(&timer.timer);
+ mode = HRTIMER_MODE_ABS;
+ __set_current_state(TASK_RUNNING);
+ destroy_hrtimer_on_stack(&timer.timer);
+
+ ktime_get_ts64(&tc);
+ entry->last_irqtime = tc.tv_nsec - oldtc.tv_nsec - sleep_ti;
+}
+
+int io_uring_hybrid_poll(struct io_kiocb *req,
+ struct io_comp_batch *iob, unsigned int poll_flags)
+{
+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+ struct io_ring_ctx *ctx = req->ctx;
+ struct hy_poll_time *hpt = req->hy_poll;
+ u32 index = req->file->f_inode->i_rdev;
+ struct iopoll_info *entry = xa_load(&ctx->poll_array, index);
+ int ret;
+
+ io_delay(hpt, entry);
+ ret = req->file->f_op->iopoll(&rw->kiocb, iob, poll_flags);
+
+ ktime_get_ts64(&hpt->iopoll_end);
+ entry->last_runtime = hpt->iopoll_end.tv_nsec - hpt->iopoll_start.tv_nsec;
+ return ret;
+}
+
int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
{
struct io_wq_work_node *pos, *start, *prev;
@@ -1145,7 +1256,9 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
if (READ_ONCE(req->iopoll_completed))
break;
- if (req->opcode == IORING_OP_URING_CMD) {
+ if (ctx->flags & IORING_SETUP_HY_POLL) {
+ ret = io_uring_hybrid_poll(req, &iob, poll_flags);
+ } else if (req->opcode == IORING_OP_URING_CMD) {
struct io_uring_cmd *ioucmd;
ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
--
2.40.1
