On Tue, Jan 07 2025, Bernd Schubert wrote:
> When the fuse-server terminates while the fuse-client or kernel
> still has queued URING_CMDs, these commands retain references
> to the struct file used by the fuse connection. This prevents
> fuse_dev_release() from being invoked, resulting in a hung mount
> point.
>
> This patch addresses the issue by making queued URING_CMDs
> cancelable, allowing fuse_dev_release() to proceed as expected
> and preventing the mount point from hanging.
>
> Signed-off-by: Bernd Schubert <bschubert@xxxxxxx>
> ---
> fs/fuse/dev.c | 2 ++
> fs/fuse/dev_uring.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++---
> fs/fuse/dev_uring_i.h | 9 +++++++
> 3 files changed, 79 insertions(+), 3 deletions(-)
>
> diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
> index afafa960d4725d9b64b22f17bf09c846219396d6..1b593b23f7b8c319ec38c7e726dabf516965500e 100644
> --- a/fs/fuse/dev.c
> +++ b/fs/fuse/dev.c
> @@ -599,8 +599,10 @@ static int fuse_request_queue_background(struct fuse_req *req)
> }
> __set_bit(FR_ISREPLY, &req->flags);
>
> +#ifdef CONFIG_FUSE_IO_URING
> if (fuse_uring_ready(fc))
> return fuse_request_queue_background_uring(fc, req);
> +#endif
I guess this should be moved to the previous patch.
Cheers,
--
Luís
>
> spin_lock(&fc->bg_lock);
> if (likely(fc->connected)) {
> diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
> index 4e4385dff9315d25aa8c37a37f1e902aec3fcd20..cdd3917b365f4040c0f147648b09af9a41e2f49e 100644
> --- a/fs/fuse/dev_uring.c
> +++ b/fs/fuse/dev_uring.c
> @@ -153,6 +153,7 @@ void fuse_uring_destruct(struct fuse_conn *fc)
>
> for (qid = 0; qid < ring->nr_queues; qid++) {
> struct fuse_ring_queue *queue = ring->queues[qid];
> + struct fuse_ring_ent *ent, *next;
>
> if (!queue)
> continue;
> @@ -162,6 +163,12 @@ void fuse_uring_destruct(struct fuse_conn *fc)
> WARN_ON(!list_empty(&queue->ent_commit_queue));
> WARN_ON(!list_empty(&queue->ent_in_userspace));
>
> + list_for_each_entry_safe(ent, next, &queue->ent_released,
> + list) {
> + list_del_init(&ent->list);
> + kfree(ent);
> + }
> +
> kfree(queue->fpq.processing);
> kfree(queue);
> ring->queues[qid] = NULL;
> @@ -245,6 +252,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
> INIT_LIST_HEAD(&queue->ent_in_userspace);
> INIT_LIST_HEAD(&queue->fuse_req_queue);
> INIT_LIST_HEAD(&queue->fuse_req_bg_queue);
> + INIT_LIST_HEAD(&queue->ent_released);
>
> queue->fpq.processing = pq;
> fuse_pqueue_init(&queue->fpq);
> @@ -283,6 +291,7 @@ static void fuse_uring_stop_fuse_req_end(struct fuse_ring_ent *ent)
> */
> static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent)
> {
> + struct fuse_ring_queue *queue = ent->queue;
> if (ent->cmd) {
> io_uring_cmd_done(ent->cmd, -ENOTCONN, 0, IO_URING_F_UNLOCKED);
> ent->cmd = NULL;
> @@ -291,8 +300,16 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent)
> if (ent->fuse_req)
> fuse_uring_stop_fuse_req_end(ent);
>
> - list_del_init(&ent->list);
> - kfree(ent);
> + /*
> + * The entry must not be freed immediately, due to access of direct
> + * pointer access of entries through IO_URING_F_CANCEL - there is a risk
> + * of race between daemon termination (which triggers IO_URING_F_CANCEL
> + * and accesses entries without checking the list state first
> + */
> + spin_lock(&queue->lock);
> + list_move(&ent->list, &queue->ent_released);
> + ent->state = FRRS_RELEASED;
> + spin_unlock(&queue->lock);
> }
>
> static void fuse_uring_stop_list_entries(struct list_head *head,
> @@ -312,6 +329,7 @@ static void fuse_uring_stop_list_entries(struct list_head *head,
> continue;
> }
>
> + ent->state = FRRS_TEARDOWN;
> list_move(&ent->list, &to_teardown);
> }
> spin_unlock(&queue->lock);
> @@ -426,6 +444,46 @@ void fuse_uring_stop_queues(struct fuse_ring *ring)
> }
> }
>
> +/*
> + * Handle IO_URING_F_CANCEL, typically should come on daemon termination.
> + *
> + * Releasing the last entry should trigger fuse_dev_release() if
> + * the daemon was terminated
> + */
> +static void fuse_uring_cancel(struct io_uring_cmd *cmd,
> + unsigned int issue_flags)
> +{
> + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);
> + struct fuse_ring_queue *queue;
> + bool need_cmd_done = false;
> +
> + /*
> + * direct access on ent - it must not be destructed as long as
> + * IO_URING_F_CANCEL might come up
> + */
> + queue = ent->queue;
> + spin_lock(&queue->lock);
> + if (ent->state == FRRS_AVAILABLE) {
> + ent->state = FRRS_USERSPACE;
> + list_move(&ent->list, &queue->ent_in_userspace);
> + need_cmd_done = true;
> + ent->cmd = NULL;
> + }
> + spin_unlock(&queue->lock);
> +
> + if (need_cmd_done) {
> + /* no queue lock to avoid lock order issues */
> + io_uring_cmd_done(cmd, -ENOTCONN, 0, issue_flags);
> + }
> +}
> +
> +static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags,
> + struct fuse_ring_ent *ring_ent)
> +{
> + uring_cmd_set_ring_ent(cmd, ring_ent);
> + io_uring_cmd_mark_cancelable(cmd, issue_flags);
> +}
> +
> /*
> * Checks for errors and stores it into the request
> */
> @@ -836,6 +894,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
> spin_unlock(&queue->lock);
>
> /* without the queue lock, as other locks are taken */
> + fuse_uring_prepare_cancel(ring_ent->cmd, issue_flags, ring_ent);
> fuse_uring_commit(ring_ent, issue_flags);
>
> /*
> @@ -885,6 +944,8 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ring_ent,
> struct fuse_conn *fc = ring->fc;
> struct fuse_iqueue *fiq = &fc->iq;
>
> + fuse_uring_prepare_cancel(ring_ent->cmd, issue_flags, ring_ent);
> +
> spin_lock(&queue->lock);
> fuse_uring_ent_avail(ring_ent, queue);
> spin_unlock(&queue->lock);
> @@ -1041,6 +1102,11 @@ int __maybe_unused fuse_uring_cmd(struct io_uring_cmd *cmd,
> return -EOPNOTSUPP;
> }
>
> + if ((unlikely(issue_flags & IO_URING_F_CANCEL))) {
> + fuse_uring_cancel(cmd, issue_flags);
> + return 0;
> + }
> +
> /* This extra SQE size holds struct fuse_uring_cmd_req */
> if (!(issue_flags & IO_URING_F_SQE128))
> return -EINVAL;
> @@ -1173,7 +1239,6 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req)
>
> if (ent) {
> struct io_uring_cmd *cmd = ent->cmd;
> -
> err = -EIO;
> if (WARN_ON_ONCE(ent->state != FRRS_FUSE_REQ))
> goto err;
> diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
> index a4271f4e55aa9d2d9b42f3d2c4095887f9563351..af2b3de829949a778d60493f36588fea67a4ba85 100644
> --- a/fs/fuse/dev_uring_i.h
> +++ b/fs/fuse/dev_uring_i.h
> @@ -28,6 +28,12 @@ enum fuse_ring_req_state {
>
> /* The ring entry is in or on the way to user space */
> FRRS_USERSPACE,
> +
> + /* The ring entry is in teardown */
> + FRRS_TEARDOWN,
> +
> + /* The ring entry is released, but not freed yet */
> + FRRS_RELEASED,
> };
>
> /** A fuse ring entry, part of the ring queue */
> @@ -79,6 +85,9 @@ struct fuse_ring_queue {
> /* entries in userspace */
> struct list_head ent_in_userspace;
>
> + /* entries that are released */
> + struct list_head ent_released;
> +
> /* fuse requests waiting for an entry slot */
> struct list_head fuse_req_queue;
>
>
> --
> 2.43.0
>
>
