* Only complie-tested right now. * There are so many duplicated code doing add/post event and then put req. Put them to common funcs io_cqring_event_posted_and_put() and io_cqring_add_event_and_put(). Signed-off-by: Bob Liu <bob.liu@xxxxxxxxxx> --- fs/io_uring.c | 145 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 299a218..816eef3 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1039,6 +1039,56 @@ static void io_double_put_req(struct io_kiocb *req) io_free_req(req); } +/* + * Add event to io_cqring and put req. + */ +static void io_cqring_add_event_and_put(struct io_kiocb *req, long ret, + int should_fail_link, bool double_put, struct io_kiocb **nxt) +{ + if (should_fail_link == 1) { + if (ret < 0 && (req->flags & REQ_F_LINK)) + req->flags |= REQ_F_FAIL_LINK; + } else if (should_fail_link == 2) { + /* Don't care about ret < 0 when should_fail_link == 2 */ + if (req->flags & REQ_F_LINK) + req->flags |= REQ_F_FAIL_LINK; + } + + io_cqring_add_event(req, ret); + + if (double_put) + io_double_put_req(req); + else { + if (nxt) + io_put_req_find_next(req, nxt); + else + io_put_req(req); + } +} + +/* + * Post event and put req. + */ +static void io_cqring_event_posted_and_put(struct io_kiocb *req, long ret, + int should_fail_link, struct io_kiocb **nxt) +{ + if (should_fail_link == 1) { + if (ret < 0 && (req->flags & REQ_F_LINK)) + req->flags |= REQ_F_FAIL_LINK; + } else if (should_fail_link == 2) { + /* Don't care about ret < 0 when should_fail_link == 2 */ + if (req->flags & REQ_F_LINK) + req->flags |= REQ_F_FAIL_LINK; + } + + io_cqring_ev_posted(req->ctx); + + if (nxt) + io_put_req_find_next(req, nxt); + else + io_put_req(req); +} + static unsigned io_cqring_events(struct io_ring_ctx *ctx, bool noflush) { struct io_rings *rings = ctx->rings; @@ -1789,8 +1839,7 @@ static int io_nop(struct io_kiocb *req) if (unlikely(ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; - io_cqring_add_event(req, 0); - io_put_req(req); + io_cqring_add_event_and_put(req, 0, 0, false, NULL); return 0; } @@ -1834,10 +1883,7 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe, end > 0 ? end : LLONG_MAX, fsync_flags & IORING_FSYNC_DATASYNC); - if (ret < 0 && (req->flags & REQ_F_LINK)) - req->flags |= REQ_F_FAIL_LINK; - io_cqring_add_event(req, ret); - io_put_req_find_next(req, nxt); + io_cqring_add_event_and_put(req, ret, 1, false, nxt); return 0; } @@ -1880,11 +1926,8 @@ static int io_sync_file_range(struct io_kiocb *req, flags = READ_ONCE(sqe->sync_range_flags); ret = sync_file_range(req->rw.ki_filp, sqe_off, sqe_len, flags); + io_cqring_add_event_and_put(req, ret, 1, false, nxt); - if (ret < 0 && (req->flags & REQ_F_LINK)) - req->flags |= REQ_F_FAIL_LINK; - io_cqring_add_event(req, ret); - io_put_req_find_next(req, nxt); return 0; } @@ -1919,10 +1962,8 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe, return ret; } - io_cqring_add_event(req, ret); - if (ret < 0 && (req->flags & REQ_F_LINK)) - req->flags |= REQ_F_FAIL_LINK; - io_put_req_find_next(req, nxt); + io_cqring_add_event_and_put(req, ret, 1, false, nxt); + return 0; } #endif @@ -1975,10 +2016,7 @@ static int io_accept(struct io_kiocb *req, const struct io_uring_sqe *sqe, } if (ret == -ERESTARTSYS) ret = -EINTR; - if (ret < 0 && (req->flags & REQ_F_LINK)) - req->flags |= REQ_F_FAIL_LINK; - io_cqring_add_event(req, ret); - io_put_req_find_next(req, nxt); + io_cqring_add_event_and_put(req, ret, 1, false, nxt); return 0; #else return -EOPNOTSUPP; @@ -2061,10 +2099,7 @@ static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe) ret = io_poll_cancel(ctx, READ_ONCE(sqe->addr)); spin_unlock_irq(&ctx->completion_lock); - io_cqring_add_event(req, ret); - if (ret < 0 && (req->flags & REQ_F_LINK)) - req->flags |= REQ_F_FAIL_LINK; - io_put_req(req); + io_cqring_add_event_and_put(req, ret, 1, false, NULL); return 0; } @@ -2118,11 +2153,7 @@ static void io_poll_complete_work(struct io_wq_work **workptr) io_poll_complete(req, mask, ret); spin_unlock_irq(&ctx->completion_lock); - io_cqring_ev_posted(ctx); - - if (ret < 0 && req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_put_req_find_next(req, &nxt); + io_cqring_event_posted_and_put(req, ret, 1, &nxt); if (nxt) *workptr = &nxt->work; } @@ -2267,10 +2298,9 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe, } spin_unlock_irq(&ctx->completion_lock); - if (mask) { - io_cqring_ev_posted(ctx); - io_put_req_find_next(req, nxt); - } + if (mask) + io_cqring_event_posted_and_put(req, 0, 0, nxt); + return ipt.error; } @@ -2308,10 +2338,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) io_commit_cqring(ctx); spin_unlock_irqrestore(&ctx->completion_lock, flags); - io_cqring_ev_posted(ctx); - if (req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_put_req(req); + io_cqring_event_posted_and_put(req, 0, 2, NULL); return HRTIMER_NORESTART; } @@ -2366,10 +2393,7 @@ static int io_timeout_remove(struct io_kiocb *req, io_cqring_fill_event(req, ret); io_commit_cqring(ctx); spin_unlock_irq(&ctx->completion_lock); - io_cqring_ev_posted(ctx); - if (ret < 0 && req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_put_req(req); + io_cqring_event_posted_and_put(req, ret, 1, NULL); return 0; } @@ -2530,11 +2554,7 @@ static void io_async_find_and_cancel(struct io_ring_ctx *ctx, io_cqring_fill_event(req, ret); io_commit_cqring(ctx); spin_unlock_irqrestore(&ctx->completion_lock, flags); - io_cqring_ev_posted(ctx); - - if (ret < 0 && (req->flags & REQ_F_LINK)) - req->flags |= REQ_F_FAIL_LINK; - io_put_req_find_next(req, nxt); + io_cqring_event_posted_and_put(req, ret, 1, nxt); } static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe, @@ -2722,12 +2742,8 @@ static void io_wq_submit_work(struct io_wq_work **workptr) /* drop submission reference */ io_put_req(req); - if (ret) { - if (req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_cqring_add_event(req, ret); - io_put_req(req); - } + if (ret) + io_cqring_add_event_and_put(req, ret, 2, false, NULL); /* if a dependent link is ready, pass it back */ if (!ret && nxt) { @@ -2870,8 +2886,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) -ETIME); io_put_req(prev); } else { - io_cqring_add_event(req, -ETIME); - io_put_req(req); + io_cqring_add_event_and_put(req, -ETIME, 0, false, NULL); } return HRTIMER_NORESTART; } @@ -2962,12 +2977,8 @@ static void __io_queue_sqe(struct io_kiocb *req) } /* and drop final reference, if we failed */ - if (ret) { - io_cqring_add_event(req, ret); - if (req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_put_req(req); - } + if (ret) + io_cqring_add_event_and_put(req, ret, 2, false, NULL); } static void io_queue_sqe(struct io_kiocb *req) @@ -2975,14 +2986,10 @@ static void io_queue_sqe(struct io_kiocb *req) int ret; ret = io_req_defer(req); - if (!ret) { + if (!ret) __io_queue_sqe(req); - } else if (ret != -EIOCBQUEUED) { - io_cqring_add_event(req, ret); - if (req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_double_put_req(req); - } + else if (ret != -EIOCBQUEUED) + io_cqring_add_event_and_put(req, ret, 2, true, NULL); } static void io_queue_link_head(struct io_kiocb *req, struct io_kiocb *shadow) @@ -3010,10 +3017,7 @@ static void io_queue_link_head(struct io_kiocb *req, struct io_kiocb *shadow) if (ret) { if (ret != -EIOCBQUEUED) { err: - io_cqring_add_event(req, ret); - if (req->flags & REQ_F_LINK) - req->flags |= REQ_F_FAIL_LINK; - io_double_put_req(req); + io_cqring_add_event_and_put(req, ret, 2, true, NULL); if (shadow) __io_free_req(shadow); return; @@ -3057,8 +3061,7 @@ static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state, ret = io_req_set_file(state, req); if (unlikely(ret)) { err_req: - io_cqring_add_event(req, ret); - io_double_put_req(req); + io_cqring_add_event_and_put(req, ret, 0, true, NULL); return; } -- 2.9.5