In preparation for expanding how we handle waits, move the actual
schedule and schedule_timeout() handling into a helper.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
io_uring/io_uring.c | 37 +++++++++++++++++++++----------------
1 file changed, 21 insertions(+), 16 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 45a2f8f3a77c..8f52acc14ebc 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2516,22 +2516,10 @@ static bool current_pending_io(void)
return percpu_counter_read_positive(&tctx->inflight);
}
-/* when returns >0, the caller should retry */
-static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
- struct io_wait_queue *iowq)
+static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+ struct io_wait_queue *iowq)
{
- int io_wait, ret;
-
- if (unlikely(READ_ONCE(ctx->check_cq)))
- return 1;
- if (unlikely(!llist_empty(&ctx->work_llist)))
- return 1;
- if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
- return 1;
- if (unlikely(task_sigpending(current)))
- return -EINTR;
- if (unlikely(io_should_wake(iowq)))
- return 0;
+ int io_wait, ret = 0;
/*
* Mark us as being in io_wait if we have pending requests, so cpufreq
@@ -2541,7 +2529,6 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
io_wait = current->in_iowait;
if (current_pending_io())
current->in_iowait = 1;
- ret = 0;
if (iowq->timeout == KTIME_MAX)
schedule();
else if (!schedule_hrtimeout(&iowq->timeout, HRTIMER_MODE_ABS))
@@ -2550,6 +2537,24 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
return ret;
}
+/* when returns >0, the caller should retry */
+static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
+ struct io_wait_queue *iowq)
+{
+ if (unlikely(READ_ONCE(ctx->check_cq)))
+ return 1;
+ if (unlikely(!llist_empty(&ctx->work_llist)))
+ return 1;
+ if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
+ return 1;
+ if (unlikely(task_sigpending(current)))
+ return -EINTR;
+ if (unlikely(io_should_wake(iowq)))
+ return 0;
+
+ return __io_cqring_wait_schedule(ctx, iowq);
+}
+
/*
* Wait until events become available, if we don't already have some. The
* application must reap them itself, as they reside on the shared cq ring.
--
2.43.0
