On 12/11/2020 23:00, Bijan Mottahedeh wrote: > Introduce IORING_REGISTER_BUFFERS_UPDATE and IORING_OP_BUFFERS_UPDATE, > consistent with file registration update. I'd prefer to not add a new opcode for each new resource. Can we have only IORING_OP_RESOURCE_UPDATE and multiplex inside? Even better if you could fit all into IORING_OP_FILES_UPDATE and then #define IORING_OP_RESOURCE_UPDATE IORING_OP_FILES_UPDATE Jens, what do you think? > > Signed-off-by: Bijan Mottahedeh <bijan.mottahedeh@xxxxxxxxxx> > --- > fs/io_uring.c | 139 +++++++++++++++++++++++++++++++++++++++++- > include/uapi/linux/io_uring.h | 8 +-- > 2 files changed, 140 insertions(+), 7 deletions(-) > > diff --git a/fs/io_uring.c b/fs/io_uring.c > index 71f6d5c..6020fd2 100644 > --- a/fs/io_uring.c > +++ b/fs/io_uring.c > @@ -1006,6 +1006,9 @@ struct io_op_def { > .work_flags = IO_WQ_WORK_MM | IO_WQ_WORK_FILES | > IO_WQ_WORK_FS | IO_WQ_WORK_BLKCG, > }, > + [IORING_OP_BUFFERS_UPDATE] = { > + .work_flags = IO_WQ_WORK_MM, > + }, > }; > > enum io_mem_account { > @@ -1025,6 +1028,9 @@ static void __io_complete_rw(struct io_kiocb *req, long res, long res2, > static int __io_sqe_files_update(struct io_ring_ctx *ctx, > struct io_uring_rsrc_update *ip, > unsigned nr_args); > +static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, > + struct io_uring_rsrc_update *up, > + unsigned nr_args); > static void __io_clean_op(struct io_kiocb *req); > static struct file *io_file_get(struct io_submit_state *state, > struct io_kiocb *req, int fd, bool fixed); > @@ -5939,6 +5945,19 @@ static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node) > percpu_ref_exit(&ref_node->refs); > kfree(ref_node); > } > + > +static int io_buffers_update_prep(struct io_kiocb *req, > + const struct io_uring_sqe *sqe) > +{ > + return io_rsrc_update_prep(req, sqe); > +} > + > +static int io_buffers_update(struct io_kiocb *req, bool force_nonblock, > + struct io_comp_state *cs) > +{ > + return io_rsrc_update(req, force_nonblock, cs, __io_sqe_buffers_update); > +} > + > static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) > { > switch (req->opcode) { > @@ -6010,11 +6029,13 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) > return io_renameat_prep(req, sqe); > case IORING_OP_UNLINKAT: > return io_unlinkat_prep(req, sqe); > + case IORING_OP_BUFFERS_UPDATE: > + return io_buffers_update_prep(req, sqe); > } > > printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", > req->opcode); > - return-EINVAL; > + return -EINVAL; > } > > static int io_req_defer_prep(struct io_kiocb *req, > @@ -6268,6 +6289,9 @@ static int io_issue_sqe(struct io_kiocb *req, bool force_nonblock, > case IORING_OP_UNLINKAT: > ret = io_unlinkat(req, force_nonblock); > break; > + case IORING_OP_BUFFERS_UPDATE: > + ret = io_buffers_update(req, force_nonblock, cs); > + break; > default: > ret = -EINVAL; > break; > @@ -8224,6 +8248,7 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu) > if (imu->acct_pages) > io_unaccount_mem(ctx, imu->nr_bvecs, ACCT_PINNED); > kvfree(imu->bvec); > + imu->bvec = NULL; > imu->nr_bvecs = 0; > } > > @@ -8441,6 +8466,7 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, > if (pret > 0) > unpin_user_pages(pages, pret); > kvfree(imu->bvec); > + imu->bvec = NULL; > goto done; > } > > @@ -8602,6 +8628,8 @@ static void io_buf_data_ref_zero(struct percpu_ref *ref) > static void io_ring_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc) > { > io_buffer_unmap(ctx, prsrc->buf); > + kvfree(prsrc->buf); > + prsrc->buf = NULL; > } > > static struct fixed_rsrc_ref_node *alloc_fixed_buf_ref_node( > @@ -8684,6 +8712,111 @@ static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, > return 0; > } > > +static inline int io_queue_buffer_removal(struct fixed_rsrc_data *data, > + struct io_mapped_ubuf *imu) > +{ > + return io_queue_rsrc_removal(data, (void *)imu); > +} > + > +static void destroy_fixed_buf_ref_node(struct fixed_rsrc_ref_node *ref_node) > +{ > + destroy_fixed_rsrc_ref_node(ref_node); > +} > + > +static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, > + struct io_uring_rsrc_update *up, > + unsigned nr_args) > +{ > + struct fixed_rsrc_data *data = ctx->buf_data; > + struct fixed_rsrc_ref_node *ref_node; > + struct io_mapped_ubuf *imu; > + struct iovec iov; > + struct iovec __user *iovs; > + struct page *last_hpage = NULL; > + __u32 done; > + int i, err; > + bool needs_switch = false; > + > + if (check_add_overflow(up->offset, nr_args, &done)) > + return -EOVERFLOW; > + if (done > ctx->nr_user_bufs) > + return -EINVAL; > + > + ref_node = alloc_fixed_buf_ref_node(ctx); > + if (IS_ERR(ref_node)) > + return PTR_ERR(ref_node); > + > + done = 0; > + iovs = u64_to_user_ptr(up->iovs); > + while (nr_args) { > + struct fixed_rsrc_table *table; > + unsigned index; > + > + err = 0; > + if (copy_from_user(&iov, &iovs[done], sizeof(iov))) { > + err = -EFAULT; > + break; > + } > + i = array_index_nospec(up->offset, ctx->nr_user_bufs); > + table = &ctx->buf_data->table[i >> IORING_BUF_TABLE_SHIFT]; > + index = i & IORING_BUF_TABLE_MASK; > + imu = &table->bufs[index]; > + if (table->bufs[index].ubuf) { > + struct io_mapped_ubuf *dup; > + dup = kmemdup(imu, sizeof(*imu), GFP_KERNEL); > + if (!dup) { > + err = -ENOMEM; > + break; > + } > + err = io_queue_buffer_removal(data, dup); > + if (err) > + break; > + memset(imu, 0, sizeof(*imu)); > + needs_switch = true; > + } > + if (!io_buffer_validate(&iov)) { > + err = io_sqe_buffer_register(ctx, &iov, imu, > + &last_hpage); > + if (err) { > + memset(imu, 0, sizeof(*imu)); > + break; > + } > + } > + nr_args--; > + done++; > + up->offset++; > + } > + > + if (needs_switch) { > + percpu_ref_kill(&data->node->refs); > + spin_lock(&data->lock); > + list_add(&ref_node->node, &data->ref_list); > + data->node = ref_node; > + spin_unlock(&data->lock); > + percpu_ref_get(&ctx->buf_data->refs); > + } else > + destroy_fixed_buf_ref_node(ref_node); > + > + return done ? done : err; > +} > + > +static int io_sqe_buffers_update(struct io_ring_ctx *ctx, void __user *arg, > + unsigned nr_args) > +{ > + struct io_uring_rsrc_update up; > + > + if (!ctx->buf_data) > + return -ENXIO; > + if (!nr_args) > + return -EINVAL; > + if (copy_from_user(&up, arg, sizeof(up))) > + return -EFAULT; > + if (up.resv) > + return -EINVAL; > + > + return __io_sqe_buffers_update(ctx, &up, nr_args); > +} > + > static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg) > { > __s32 __user *fds = arg; > @@ -9961,6 +10094,7 @@ static bool io_register_op_must_quiesce(int op) > switch (op) { > case IORING_UNREGISTER_FILES: > case IORING_REGISTER_FILES_UPDATE: > + case IORING_REGISTER_BUFFERS_UPDATE: > case IORING_REGISTER_PROBE: > case IORING_REGISTER_PERSONALITY: > case IORING_UNREGISTER_PERSONALITY: > @@ -10036,6 +10170,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, > break; > ret = io_sqe_buffers_unregister(ctx); > break; > + case IORING_REGISTER_BUFFERS_UPDATE: > + ret = io_sqe_buffers_update(ctx, arg, nr_args); > + break; > case IORING_REGISTER_FILES: > ret = io_sqe_files_register(ctx, arg, nr_args); > break; > diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h > index 87f0f56..17682b5 100644 > --- a/include/uapi/linux/io_uring.h > +++ b/include/uapi/linux/io_uring.h > @@ -137,6 +137,7 @@ enum { > IORING_OP_SHUTDOWN, > IORING_OP_RENAMEAT, > IORING_OP_UNLINKAT, > + IORING_OP_BUFFERS_UPDATE, > > /* this goes last, obviously */ > IORING_OP_LAST, > @@ -279,17 +280,12 @@ enum { > IORING_UNREGISTER_PERSONALITY = 10, > IORING_REGISTER_RESTRICTIONS = 11, > IORING_REGISTER_ENABLE_RINGS = 12, > + IORING_REGISTER_BUFFERS_UPDATE = 13, > > /* this goes last */ > IORING_REGISTER_LAST > }; > > -struct io_uring_files_update { > - __u32 offset; > - __u32 resv; > - __aligned_u64 /* __s32 * */ fds; > -}; > - > struct io_uring_rsrc_update { > __u32 offset; > __u32 resv; > -- Pavel Begunkov