io_run_task_work() does some extra work for safety, that isn't actually
needed in many cases, particularly when it's known that current is not
exiting and in the TASK_RUNNING state, like in the beginning of a
syscall.
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
fs/io_uring.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 080fac4d4543..45b49273df8b 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2227,6 +2227,16 @@ static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
return io_put_kbuf(req, kbuf);
}
+/* only safe when TASK_RUNNING and it's not PF_EXITING */
+static inline bool __io_run_task_work(void)
+{
+ if (current->task_works) {
+ task_work_run();
+ return true;
+ }
+ return false;
+}
+
static inline bool io_run_task_work(void)
{
/*
@@ -6729,7 +6739,7 @@ static int io_sq_thread(void *data)
}
if (sqt_spin || !time_after(jiffies, timeout)) {
- io_run_task_work();
+ __io_run_task_work();
cond_resched();
if (sqt_spin)
timeout = jiffies + sqd->sq_thread_idle;
@@ -6870,7 +6880,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
io_cqring_overflow_flush(ctx, false, NULL, NULL);
if (io_cqring_events(ctx) >= min_events)
return 0;
- if (!io_run_task_work())
+ if (!__io_run_task_work())
break;
} while (1);
@@ -9117,7 +9127,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
struct fd f;
long ret;
- io_run_task_work();
+ __io_run_task_work();
if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
--
2.24.0
