If a requests is forwarded into io-wq, there is a good chance it hasn't
been refcounted yet and we can save one req_ref_get() by setting the
refcount number to the right value directly.
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
fs/io_uring.c | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 51c4166f68b5..0d9a443d4987 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1115,14 +1115,19 @@ static inline void req_ref_get(struct io_kiocb *req)
atomic_inc(&req->refs);
}
-static inline void io_req_refcount(struct io_kiocb *req)
+static inline void __io_req_refcount(struct io_kiocb *req, int nr)
{
if (!(req->flags & REQ_F_REFCOUNT)) {
req->flags |= REQ_F_REFCOUNT;
- atomic_set(&req->refs, 1);
+ atomic_set(&req->refs, nr);
}
}
+static inline void io_req_refcount(struct io_kiocb *req)
+{
+ __io_req_refcount(req, 1);
+}
+
static inline void io_req_set_rsrc_node(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
@@ -6311,9 +6316,11 @@ static void io_wq_submit_work(struct io_wq_work *work)
struct io_kiocb *timeout;
int ret = 0;
- io_req_refcount(req);
- /* will be dropped by ->io_free_work() after returning to io-wq */
- req_ref_get(req);
+ /* one will be dropped by ->io_free_work() after returning to io-wq */
+ if (!(req->flags & REQ_F_REFCOUNT))
+ __io_req_refcount(req, 2);
+ else
+ req_ref_get(req);
timeout = io_prep_linked_timeout(req);
if (timeout)
--
2.32.0
