We need a task context to post CQEs but using wq is too expensive. Try to complete notifiers using task_work and fall back to wq if fails. Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx> --- fs/io_uring.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 422ff835bf36..9ade0ea8552b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -384,6 +384,8 @@ struct io_notif { /* hook into ctx->notif_list and ctx->notif_list_locked */ struct list_head cache_node; + /* complete via tw if ->task is non-NULL, fallback to wq otherwise */ + struct task_struct *task; union { struct callback_head task_work; struct work_struct commit_work; @@ -2802,6 +2804,11 @@ static void __io_notif_complete_tw(struct callback_head *cb) struct io_notif *notif = container_of(cb, struct io_notif, task_work); struct io_ring_ctx *ctx = notif->ctx; + if (likely(notif->task)) { + io_put_task(notif->task, 1); + notif->task = NULL; + } + spin_lock(&ctx->completion_lock); io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq); @@ -2835,6 +2842,14 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, if (!refcount_dec_and_test(&uarg->refcnt)) return; + + if (likely(notif->task)) { + init_task_work(¬if->task_work, __io_notif_complete_tw); + if (likely(!task_work_add(notif->task, ¬if->task_work, + TWA_SIGNAL))) + return; + } + INIT_WORK(¬if->commit_work, io_notif_complete_wq); queue_work(system_unbound_wq, ¬if->commit_work); } @@ -2946,8 +2961,12 @@ static __cold int io_notif_unregister(struct io_ring_ctx *ctx) for (i = 0; i < ctx->nr_notif_slots; i++) { struct io_notif_slot *slot = &ctx->notif_slots[i]; - if (slot->notif) + if (slot->notif) { + WARN_ON_ONCE(slot->notif->task); + + slot->notif->task = NULL; io_notif_slot_flush(slot); + } } kvfree(ctx->notif_slots); -- 2.36.1