On 10/27/22 8:44 AM, Dylan Yudaken wrote:
> It is possible for tw to lock the ring, and this was not propogated out to
> io_run_local_work. This can cause an unlock to be missed.
>
> Instead pass a pointer to locked into __io_run_local_work.
>
> Fixes: 8ac5d85a89b4 ("io_uring: add local task_work run helper that is entered locked")
> Signed-off-by: Dylan Yudaken <dylany@xxxxxxxx>
> ---
> io_uring/io_uring.c | 8 ++++----
> io_uring/io_uring.h | 12 ++++++++++--
> 2 files changed, 14 insertions(+), 6 deletions(-)
>
> diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
> index 8a0ce7379e89..ac8c488e3077 100644
> --- a/io_uring/io_uring.c
> +++ b/io_uring/io_uring.c
> @@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
> }
> }
>
> -int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
> +int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked)
> {
> struct llist_node *node;
> struct llist_node fake;
> @@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
> struct io_kiocb *req = container_of(node, struct io_kiocb,
> io_task_work.node);
> prefetch(container_of(next, struct io_kiocb, io_task_work.node));
> - req->io_task_work.func(req, &locked);
> + req->io_task_work.func(req, locked);
> ret++;
> node = next;
> }
> @@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
> goto again;
> }
>
> - if (locked)
> + if (*locked)
> io_submit_flush_completions(ctx);
> trace_io_uring_local_work_run(ctx, ret, loops);
> return ret;
> @@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx)
>
> __set_current_state(TASK_RUNNING);
> locked = mutex_trylock(&ctx->uring_lock);
> - ret = __io_run_local_work(ctx, locked);
> + ret = __io_run_local_work(ctx, &locked);
> if (locked)
> mutex_unlock(&ctx->uring_lock);
>
> diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
> index ef77d2aa3172..331ec2869212 100644
> --- a/io_uring/io_uring.h
> +++ b/io_uring/io_uring.h
> @@ -27,7 +27,7 @@ enum {
> struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
> bool io_req_cqe_overflow(struct io_kiocb *req);
> int io_run_task_work_sig(struct io_ring_ctx *ctx);
> -int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
> +int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked);
> int io_run_local_work(struct io_ring_ctx *ctx);
> void io_req_complete_failed(struct io_kiocb *req, s32 res);
> void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
> @@ -277,9 +277,17 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
>
> static inline int io_run_local_work_locked(struct io_ring_ctx *ctx)
> {
> + bool locked;
> + int ret;
> +
> if (llist_empty(&ctx->work_llist))
> return 0;
> - return __io_run_local_work(ctx, true);
> +
> + locked = true;
> + ret = __io_run_local_work(ctx, &locked);
> + if (WARN_ON(!locked))
> + mutex_lock(&ctx->uring_lock);
> + return ret;
> }
If you think warning on !locked is a good idea, it should be a
WARN_ON_ONCE(). Or is this leftover debugging?
--
Jens Axboe
