Because of need_resched() check, io_uring_enter() -> io_iopoll_check()
can return 0 even if @min_complete wasn't satisfied. If that's the
case, __io_uring_get_cqe() sets submit=0 and wait_nr=0, disabling
setting IORING_ENTER_GETEVENTS as well. So, it goes crazy calling
io_uring_enter() in a loop, not actually submitting nor polling.
Set @wait_nr based on actual number of CQEs ready. It doesn't manifest
extra CQEs if any, thus implements __io_uring_cq_ready() with relaxed
semantics.
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
src/queue.c | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/src/queue.c b/src/queue.c
index 14a0777..d824cfd 100644
--- a/src/queue.c
+++ b/src/queue.c
@@ -32,6 +32,19 @@ static inline bool sq_ring_needs_enter(struct io_uring *ring,
return false;
}
+static inline unsigned int __io_uring_cq_ready(struct io_uring *ring)
+{
+ return io_uring_smp_load_relaxed(ring->cq.ktail) - *ring->cq.khead;
+}
+
+static inline unsigned int io_adjust_wait_nr(struct io_uring *ring,
+ unsigned int to_wait)
+{
+ unsigned int ready = __io_uring_cq_ready(ring);
+
+ return (to_wait <= ready) ? 0 : (to_wait - ready);
+}
+
int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
unsigned submit, unsigned wait_nr, sigset_t *sigmask)
{
@@ -60,7 +73,8 @@ int __io_uring_get_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr,
err = -errno;
} else if (ret == (int)submit) {
submit = 0;
- wait_nr = 0;
+ if (to_wait)
+ wait_nr = io_adjust_wait_nr(ring, to_wait);
} else {
submit -= ret;
}
--
2.24.0
