This is a note to let you know that I've just added the patch titled
io_uring: clean cqe filling functions
to the 5.15-stable tree which can be found at:
http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary
The filename of the patch is:
io_uring-clean-cqe-filling-functions.patch
and it can be found in the queue-5.15 subdirectory.
If you, or anyone else, feels it should not be added to the stable tree,
please let <stable@xxxxxxxxxxxxxxx> know about it.
>From foo@baz Thu Sep 1 11:59:31 AM CEST 2022
From: Pavel Begunkov <asml.silence@xxxxxxxxx>
Date: Mon, 29 Aug 2022 14:30:13 +0100
Subject: io_uring: clean cqe filling functions
To: stable@xxxxxxxxxxxxxxx
Cc: Jens Axboe <axboe@xxxxxxxxx>, asml.silence@xxxxxxxxx
Message-ID: <ca69eee269c72bb415d2863408355504a10ab8c8.1661594698.git.asml.silence@xxxxxxxxx>
From: Pavel Begunkov <asml.silence@xxxxxxxxx>
[ upstream commmit 913a571affedd17239c4d4ea90c8874b32fc2191 ]
Split io_cqring_fill_event() into a couple of more targeted functions.
The first on is io_fill_cqe_aux() for completions that are not
associated with request completions and doing the ->cq_extra accounting.
Examples are additional CQEs from multishot poll and rsrc notifications.
The second is io_fill_cqe_req(), should be called when it's a normal
request completion. Nothing more to it at the moment, will be used in
later patches.
The last one is inlined __io_fill_cqe() for a finer grained control,
should be used with caution and in hottest places.
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
Link: https://lore.kernel.org/r/59a9117a4a44fc9efcf04b3afa51e0d080f5943c.1636559119.git.asml.silence@xxxxxxxxx
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
[pavel: backport]
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
---
fs/io_uring.c | 57 +++++++++++++++++++++++++++++----------------------------
1 file changed, 29 insertions(+), 28 deletions(-)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1079,8 +1079,8 @@ static void io_uring_try_cancel_requests
bool cancel_all);
static void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd);
-static bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
- s32 res, u32 cflags);
+static void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags);
+
static void io_put_req(struct io_kiocb *req);
static void io_put_req_deferred(struct io_kiocb *req);
static void io_dismantle_req(struct io_kiocb *req);
@@ -1515,7 +1515,7 @@ static void io_kill_timeout(struct io_ki
atomic_set(&req->ctx->cq_timeouts,
atomic_read(&req->ctx->cq_timeouts) + 1);
list_del_init(&req->timeout.list);
- io_cqring_fill_event(req->ctx, req->user_data, status, 0);
+ io_fill_cqe_req(req, status, 0);
io_put_req_deferred(req);
}
}
@@ -1790,8 +1790,8 @@ static bool io_cqring_event_overflow(str
return true;
}
-static inline bool __io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
- s32 res, u32 cflags)
+static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,
+ s32 res, u32 cflags)
{
struct io_uring_cqe *cqe;
@@ -1812,11 +1812,16 @@ static inline bool __io_cqring_fill_even
return io_cqring_event_overflow(ctx, user_data, res, cflags);
}
-/* not as hot to bloat with inlining */
-static noinline bool io_cqring_fill_event(struct io_ring_ctx *ctx, u64 user_data,
- s32 res, u32 cflags)
+static noinline void io_fill_cqe_req(struct io_kiocb *req, s32 res, u32 cflags)
+{
+ __io_fill_cqe(req->ctx, req->user_data, res, cflags);
+}
+
+static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
+ s32 res, u32 cflags)
{
- return __io_cqring_fill_event(ctx, user_data, res, cflags);
+ ctx->cq_extra++;
+ return __io_fill_cqe(ctx, user_data, res, cflags);
}
static void io_req_complete_post(struct io_kiocb *req, s32 res,
@@ -1825,7 +1830,7 @@ static void io_req_complete_post(struct
struct io_ring_ctx *ctx = req->ctx;
spin_lock(&ctx->completion_lock);
- __io_cqring_fill_event(ctx, req->user_data, res, cflags);
+ __io_fill_cqe(ctx, req->user_data, res, cflags);
/*
* If we're the last reference to this request, add to our locked
* free_list cache.
@@ -2051,8 +2056,7 @@ static bool io_kill_linked_timeout(struc
link->timeout.head = NULL;
if (hrtimer_try_to_cancel(&io->timer) != -1) {
list_del(&link->timeout.list);
- io_cqring_fill_event(link->ctx, link->user_data,
- -ECANCELED, 0);
+ io_fill_cqe_req(link, -ECANCELED, 0);
io_put_req_deferred(link);
return true;
}
@@ -2076,7 +2080,7 @@ static void io_fail_links(struct io_kioc
link->link = NULL;
trace_io_uring_fail_link(req, link);
- io_cqring_fill_event(link->ctx, link->user_data, res, 0);
+ io_fill_cqe_req(link, res, 0);
io_put_req_deferred(link);
link = nxt;
}
@@ -2093,8 +2097,7 @@ static bool io_disarm_next(struct io_kio
req->flags &= ~REQ_F_ARM_LTIMEOUT;
if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
io_remove_next_linked(req);
- io_cqring_fill_event(link->ctx, link->user_data,
- -ECANCELED, 0);
+ io_fill_cqe_req(link, -ECANCELED, 0);
io_put_req_deferred(link);
posted = true;
}
@@ -2370,8 +2373,8 @@ static void io_submit_flush_completions(
for (i = 0; i < nr; i++) {
struct io_kiocb *req = state->compl_reqs[i];
- __io_cqring_fill_event(ctx, req->user_data, req->result,
- req->compl.cflags);
+ __io_fill_cqe(ctx, req->user_data, req->result,
+ req->compl.cflags);
}
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
@@ -2482,8 +2485,7 @@ static void io_iopoll_complete(struct io
req = list_first_entry(done, struct io_kiocb, inflight_entry);
list_del(&req->inflight_entry);
- __io_cqring_fill_event(ctx, req->user_data, req->result,
- io_put_rw_kbuf(req));
+ io_fill_cqe_req(req, req->result, io_put_rw_kbuf(req));
(*nr_events)++;
if (req_ref_put_and_test(req))
@@ -5413,13 +5415,13 @@ static bool __io_poll_complete(struct io
}
if (req->poll.events & EPOLLONESHOT)
flags = 0;
- if (!io_cqring_fill_event(ctx, req->user_data, error, flags)) {
+
+ if (!(flags & IORING_CQE_F_MORE)) {
+ io_fill_cqe_req(req, error, flags);
+ } else if (!io_fill_cqe_aux(ctx, req->user_data, error, flags)) {
req->poll.events |= EPOLLONESHOT;
flags = 0;
}
- if (flags & IORING_CQE_F_MORE)
- ctx->cq_extra++;
-
return !(flags & IORING_CQE_F_MORE);
}
@@ -5746,9 +5748,9 @@ static bool io_poll_remove_one(struct io
do_complete = __io_poll_remove_one(req, io_poll_get_single(req), true);
if (do_complete) {
- io_cqring_fill_event(req->ctx, req->user_data, -ECANCELED, 0);
- io_commit_cqring(req->ctx);
req_set_fail(req);
+ io_fill_cqe_req(req, -ECANCELED, 0);
+ io_commit_cqring(req->ctx);
io_put_req_deferred(req);
}
return do_complete;
@@ -6045,7 +6047,7 @@ static int io_timeout_cancel(struct io_r
return PTR_ERR(req);
req_set_fail(req);
- io_cqring_fill_event(ctx, req->user_data, -ECANCELED, 0);
+ io_fill_cqe_req(req, -ECANCELED, 0);
io_put_req_deferred(req);
return 0;
}
@@ -8271,8 +8273,7 @@ static void __io_rsrc_put_work(struct io
io_ring_submit_lock(ctx, lock_ring);
spin_lock(&ctx->completion_lock);
- io_cqring_fill_event(ctx, prsrc->tag, 0, 0);
- ctx->cq_extra++;
+ io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
io_commit_cqring(ctx);
spin_unlock(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
Patches currently in stable-queue which might be from asml.silence@xxxxxxxxx are
queue-5.15/io_uring-fix-wrong-arm_poll-error-handling.patch
queue-5.15/io_uring-poll-rework.patch
queue-5.15/io_uring-remove-unused-function-req_ref_put.patch
queue-5.15/io_uring-fail-links-when-poll-fails.patch
queue-5.15/io_uring-inline-io_poll_complete.patch
queue-5.15/io_uring-bump-poll-refs-to-full-31-bits.patch
queue-5.15/io_uring-clean-cqe-filling-functions.patch
queue-5.15/io_uring-fix-uaf-due-to-missing-pollfree-handling.patch
queue-5.15/io_uring-correct-fill-events-helpers-types.patch
queue-5.15/io_uring-refactor-poll-update.patch
queue-5.15/io_uring-kill-poll-linking-optimisation.patch
queue-5.15/io_uring-remove-poll-entry-from-list-when-canceling-all.patch
queue-5.15/io_uring-move-common-poll-bits.patch
