The other less frequently used fields are now located in a secondary and
cold struct, called `io_cold_def`.
This is the size for the structs:
Before: io_issue_def = 56 bytes
After: io_issue_def = 24 bytes; io_cold_def = 40 bytes
Signed-off-by: Breno Leitao <leitao@xxxxxxxxxx>
---
io_uring/io_uring.c | 15 +-
io_uring/opdef.c | 327 ++++++++++++++++++++++++++++++--------------
io_uring/opdef.h | 9 +-
io_uring/rw.c | 2 +-
4 files changed, 238 insertions(+), 115 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index ac7868ec9be2..0be66b026a7f 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -980,7 +980,7 @@ void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
void io_req_defer_failed(struct io_kiocb *req, s32 res)
__must_hold(&ctx->uring_lock)
{
- const struct io_issue_def *def = &io_issue_defs[req->opcode];
+ const struct io_cold_def *def = &io_cold_defs[req->opcode];
lockdep_assert_held(&req->ctx->uring_lock);
@@ -1708,8 +1708,8 @@ unsigned int io_file_get_flags(struct file *file)
bool io_alloc_async_data(struct io_kiocb *req)
{
- WARN_ON_ONCE(!io_issue_defs[req->opcode].async_size);
- req->async_data = kmalloc(io_issue_defs[req->opcode].async_size, GFP_KERNEL);
+ WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size);
+ req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL);
if (req->async_data) {
req->flags |= REQ_F_ASYNC_DATA;
return false;
@@ -1719,20 +1719,21 @@ bool io_alloc_async_data(struct io_kiocb *req)
int io_req_prep_async(struct io_kiocb *req)
{
+ const struct io_cold_def *cdef = &io_cold_defs[req->opcode];
const struct io_issue_def *def = &io_issue_defs[req->opcode];
/* assign early for deferred execution for non-fixed file */
if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE))
req->file = io_file_get_normal(req, req->cqe.fd);
- if (!def->prep_async)
+ if (!cdef->prep_async)
return 0;
if (WARN_ON_ONCE(req_has_async_data(req)))
return -EFAULT;
- if (!io_issue_defs[req->opcode].manual_alloc) {
+ if (!def->manual_alloc) {
if (io_alloc_async_data(req))
return -EAGAIN;
}
- return def->prep_async(req);
+ return cdef->prep_async(req);
}
static u32 io_get_sequence(struct io_kiocb *req)
@@ -1801,7 +1802,7 @@ static void io_clean_op(struct io_kiocb *req)
}
if (req->flags & REQ_F_NEED_CLEANUP) {
- const struct io_issue_def *def = &io_issue_defs[req->opcode];
+ const struct io_cold_def *def = &io_cold_defs[req->opcode];
if (def->cleanup)
def->cleanup(req);
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index 3c95e70a625e..5238ecd7af6a 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -50,7 +50,6 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_NOP] = {
.audit_skip = 1,
.iopoll = 1,
- .name = "NOP",
.prep = io_nop_prep,
.issue = io_nop,
},
@@ -64,13 +63,8 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = sizeof(struct io_async_rw),
- .name = "READV",
.prep = io_prep_rw,
.issue = io_read,
- .prep_async = io_readv_prep_async,
- .cleanup = io_readv_writev_cleanup,
- .fail = io_rw_fail,
},
[IORING_OP_WRITEV] = {
.needs_file = 1,
@@ -82,18 +76,12 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = sizeof(struct io_async_rw),
- .name = "WRITEV",
.prep = io_prep_rw,
.issue = io_write,
- .prep_async = io_writev_prep_async,
- .cleanup = io_readv_writev_cleanup,
- .fail = io_rw_fail,
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
.audit_skip = 1,
- .name = "FSYNC",
.prep = io_fsync_prep,
.issue = io_fsync,
},
@@ -106,11 +94,8 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = sizeof(struct io_async_rw),
- .name = "READ_FIXED",
.prep = io_prep_rw,
.issue = io_read,
- .fail = io_rw_fail,
},
[IORING_OP_WRITE_FIXED] = {
.needs_file = 1,
@@ -122,30 +107,24 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = sizeof(struct io_async_rw),
- .name = "WRITE_FIXED",
.prep = io_prep_rw,
.issue = io_write,
- .fail = io_rw_fail,
},
[IORING_OP_POLL_ADD] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
- .name = "POLL_ADD",
.prep = io_poll_add_prep,
.issue = io_poll_add,
},
[IORING_OP_POLL_REMOVE] = {
.audit_skip = 1,
- .name = "POLL_REMOVE",
.prep = io_poll_remove_prep,
.issue = io_poll_remove,
},
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
.audit_skip = 1,
- .name = "SYNC_FILE_RANGE",
.prep = io_sfr_prep,
.issue = io_sync_file_range,
},
@@ -155,14 +134,9 @@ const struct io_issue_def io_issue_defs[] = {
.pollout = 1,
.ioprio = 1,
.manual_alloc = 1,
- .name = "SENDMSG",
#if defined(CONFIG_NET)
- .async_size = sizeof(struct io_async_msghdr),
.prep = io_sendmsg_prep,
.issue = io_sendmsg,
- .prep_async = io_sendmsg_prep_async,
- .cleanup = io_sendmsg_recvmsg_cleanup,
- .fail = io_sendrecv_fail,
#else
.prep = io_eopnotsupp_prep,
#endif
@@ -174,29 +148,21 @@ const struct io_issue_def io_issue_defs[] = {
.buffer_select = 1,
.ioprio = 1,
.manual_alloc = 1,
- .name = "RECVMSG",
#if defined(CONFIG_NET)
- .async_size = sizeof(struct io_async_msghdr),
.prep = io_recvmsg_prep,
.issue = io_recvmsg,
- .prep_async = io_recvmsg_prep_async,
- .cleanup = io_sendmsg_recvmsg_cleanup,
- .fail = io_sendrecv_fail,
#else
.prep = io_eopnotsupp_prep,
#endif
},
[IORING_OP_TIMEOUT] = {
.audit_skip = 1,
- .async_size = sizeof(struct io_timeout_data),
- .name = "TIMEOUT",
.prep = io_timeout_prep,
.issue = io_timeout,
},
[IORING_OP_TIMEOUT_REMOVE] = {
/* used by timeout updates' prep() */
.audit_skip = 1,
- .name = "TIMEOUT_REMOVE",
.prep = io_timeout_remove_prep,
.issue = io_timeout_remove,
},
@@ -206,7 +172,6 @@ const struct io_issue_def io_issue_defs[] = {
.pollin = 1,
.poll_exclusive = 1,
.ioprio = 1, /* used for flags */
- .name = "ACCEPT",
#if defined(CONFIG_NET)
.prep = io_accept_prep,
.issue = io_accept,
@@ -216,14 +181,11 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_ASYNC_CANCEL] = {
.audit_skip = 1,
- .name = "ASYNC_CANCEL",
.prep = io_async_cancel_prep,
.issue = io_async_cancel,
},
[IORING_OP_LINK_TIMEOUT] = {
.audit_skip = 1,
- .async_size = sizeof(struct io_timeout_data),
- .name = "LINK_TIMEOUT",
.prep = io_link_timeout_prep,
.issue = io_no_issue,
},
@@ -231,46 +193,36 @@ const struct io_issue_def io_issue_defs[] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
- .name = "CONNECT",
#if defined(CONFIG_NET)
- .async_size = sizeof(struct io_async_connect),
.prep = io_connect_prep,
.issue = io_connect,
- .prep_async = io_connect_prep_async,
#else
.prep = io_eopnotsupp_prep,
#endif
},
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
- .name = "FALLOCATE",
.prep = io_fallocate_prep,
.issue = io_fallocate,
},
[IORING_OP_OPENAT] = {
- .name = "OPENAT",
.prep = io_openat_prep,
.issue = io_openat,
- .cleanup = io_open_cleanup,
},
[IORING_OP_CLOSE] = {
- .name = "CLOSE",
.prep = io_close_prep,
.issue = io_close,
},
[IORING_OP_FILES_UPDATE] = {
.audit_skip = 1,
.iopoll = 1,
- .name = "FILES_UPDATE",
.prep = io_files_update_prep,
.issue = io_files_update,
},
[IORING_OP_STATX] = {
.audit_skip = 1,
- .name = "STATX",
.prep = io_statx_prep,
.issue = io_statx,
- .cleanup = io_statx_cleanup,
},
[IORING_OP_READ] = {
.needs_file = 1,
@@ -282,11 +234,8 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = sizeof(struct io_async_rw),
- .name = "READ",
.prep = io_prep_rw,
.issue = io_read,
- .fail = io_rw_fail,
},
[IORING_OP_WRITE] = {
.needs_file = 1,
@@ -298,21 +247,16 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = sizeof(struct io_async_rw),
- .name = "WRITE",
.prep = io_prep_rw,
.issue = io_write,
- .fail = io_rw_fail,
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
.audit_skip = 1,
- .name = "FADVISE",
.prep = io_fadvise_prep,
.issue = io_fadvise,
},
[IORING_OP_MADVISE] = {
- .name = "MADVISE",
.prep = io_madvise_prep,
.issue = io_madvise,
},
@@ -323,13 +267,9 @@ const struct io_issue_def io_issue_defs[] = {
.audit_skip = 1,
.ioprio = 1,
.manual_alloc = 1,
- .name = "SEND",
#if defined(CONFIG_NET)
- .async_size = sizeof(struct io_async_msghdr),
.prep = io_sendmsg_prep,
.issue = io_send,
- .fail = io_sendrecv_fail,
- .prep_async = io_send_prep_async,
#else
.prep = io_eopnotsupp_prep,
#endif
@@ -341,25 +281,20 @@ const struct io_issue_def io_issue_defs[] = {
.buffer_select = 1,
.audit_skip = 1,
.ioprio = 1,
- .name = "RECV",
#if defined(CONFIG_NET)
.prep = io_recvmsg_prep,
.issue = io_recv,
- .fail = io_sendrecv_fail,
#else
.prep = io_eopnotsupp_prep,
#endif
},
[IORING_OP_OPENAT2] = {
- .name = "OPENAT2",
.prep = io_openat2_prep,
.issue = io_openat2,
- .cleanup = io_open_cleanup,
},
[IORING_OP_EPOLL_CTL] = {
.unbound_nonreg_file = 1,
.audit_skip = 1,
- .name = "EPOLL",
#if defined(CONFIG_EPOLL)
.prep = io_epoll_ctl_prep,
.issue = io_epoll_ctl,
@@ -372,21 +307,18 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
- .name = "SPLICE",
.prep = io_splice_prep,
.issue = io_splice,
},
[IORING_OP_PROVIDE_BUFFERS] = {
.audit_skip = 1,
.iopoll = 1,
- .name = "PROVIDE_BUFFERS",
.prep = io_provide_buffers_prep,
.issue = io_provide_buffers,
},
[IORING_OP_REMOVE_BUFFERS] = {
.audit_skip = 1,
.iopoll = 1,
- .name = "REMOVE_BUFFERS",
.prep = io_remove_buffers_prep,
.issue = io_remove_buffers,
},
@@ -395,13 +327,11 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
- .name = "TEE",
.prep = io_tee_prep,
.issue = io_tee,
},
[IORING_OP_SHUTDOWN] = {
.needs_file = 1,
- .name = "SHUTDOWN",
#if defined(CONFIG_NET)
.prep = io_shutdown_prep,
.issue = io_shutdown,
@@ -410,72 +340,51 @@ const struct io_issue_def io_issue_defs[] = {
#endif
},
[IORING_OP_RENAMEAT] = {
- .name = "RENAMEAT",
.prep = io_renameat_prep,
.issue = io_renameat,
- .cleanup = io_renameat_cleanup,
},
[IORING_OP_UNLINKAT] = {
- .name = "UNLINKAT",
.prep = io_unlinkat_prep,
.issue = io_unlinkat,
- .cleanup = io_unlinkat_cleanup,
},
[IORING_OP_MKDIRAT] = {
- .name = "MKDIRAT",
.prep = io_mkdirat_prep,
.issue = io_mkdirat,
- .cleanup = io_mkdirat_cleanup,
},
[IORING_OP_SYMLINKAT] = {
- .name = "SYMLINKAT",
.prep = io_symlinkat_prep,
.issue = io_symlinkat,
- .cleanup = io_link_cleanup,
},
[IORING_OP_LINKAT] = {
- .name = "LINKAT",
.prep = io_linkat_prep,
.issue = io_linkat,
- .cleanup = io_link_cleanup,
},
[IORING_OP_MSG_RING] = {
.needs_file = 1,
.iopoll = 1,
- .name = "MSG_RING",
.prep = io_msg_ring_prep,
.issue = io_msg_ring,
- .cleanup = io_msg_ring_cleanup,
},
[IORING_OP_FSETXATTR] = {
.needs_file = 1,
- .name = "FSETXATTR",
.prep = io_fsetxattr_prep,
.issue = io_fsetxattr,
- .cleanup = io_xattr_cleanup,
},
[IORING_OP_SETXATTR] = {
- .name = "SETXATTR",
.prep = io_setxattr_prep,
.issue = io_setxattr,
- .cleanup = io_xattr_cleanup,
},
[IORING_OP_FGETXATTR] = {
.needs_file = 1,
- .name = "FGETXATTR",
.prep = io_fgetxattr_prep,
.issue = io_fgetxattr,
- .cleanup = io_xattr_cleanup,
},
[IORING_OP_GETXATTR] = {
- .name = "GETXATTR",
.prep = io_getxattr_prep,
.issue = io_getxattr,
- .cleanup = io_xattr_cleanup,
},
[IORING_OP_SOCKET] = {
.audit_skip = 1,
- .name = "SOCKET",
#if defined(CONFIG_NET)
.prep = io_socket_prep,
.issue = io_socket,
@@ -486,16 +395,12 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_URING_CMD] = {
.needs_file = 1,
.plug = 1,
- .name = "URING_CMD",
.iopoll = 1,
.iopoll_queue = 1,
- .async_size = uring_cmd_pdu_size(1),
.prep = io_uring_cmd_prep,
.issue = io_uring_cmd,
- .prep_async = io_uring_cmd_prep_async,
},
[IORING_OP_SEND_ZC] = {
- .name = "SEND_ZC",
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
@@ -503,32 +408,243 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.manual_alloc = 1,
#if defined(CONFIG_NET)
- .async_size = sizeof(struct io_async_msghdr),
.prep = io_send_zc_prep,
.issue = io_send_zc,
- .prep_async = io_send_prep_async,
- .cleanup = io_send_zc_cleanup,
- .fail = io_sendrecv_fail,
#else
.prep = io_eopnotsupp_prep,
#endif
},
[IORING_OP_SENDMSG_ZC] = {
- .name = "SENDMSG_ZC",
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.ioprio = 1,
.manual_alloc = 1,
#if defined(CONFIG_NET)
- .async_size = sizeof(struct io_async_msghdr),
.prep = io_send_zc_prep,
.issue = io_sendmsg_zc,
+#else
+ .prep = io_eopnotsupp_prep,
+#endif
+ },
+};
+
+
+const struct io_cold_def io_cold_defs[] = {
+ [IORING_OP_NOP] = {
+ .name = "NOP",
+ },
+ [IORING_OP_READV] = {
+ .async_size = sizeof(struct io_async_rw),
+ .name = "READV",
+ .prep_async = io_readv_prep_async,
+ .cleanup = io_readv_writev_cleanup,
+ .fail = io_rw_fail,
+ },
+ [IORING_OP_WRITEV] = {
+ .async_size = sizeof(struct io_async_rw),
+ .name = "WRITEV",
+ .prep_async = io_writev_prep_async,
+ .cleanup = io_readv_writev_cleanup,
+ .fail = io_rw_fail,
+ },
+ [IORING_OP_FSYNC] = {
+ .name = "FSYNC",
+ },
+ [IORING_OP_READ_FIXED] = {
+ .async_size = sizeof(struct io_async_rw),
+ .name = "READ_FIXED",
+ .fail = io_rw_fail,
+ },
+ [IORING_OP_WRITE_FIXED] = {
+ .async_size = sizeof(struct io_async_rw),
+ .name = "WRITE_FIXED",
+ .fail = io_rw_fail,
+ },
+ [IORING_OP_POLL_ADD] = {
+ .name = "POLL_ADD",
+ },
+ [IORING_OP_POLL_REMOVE] = {
+ .name = "POLL_REMOVE",
+ },
+ [IORING_OP_SYNC_FILE_RANGE] = {
+ .name = "SYNC_FILE_RANGE",
+ },
+ [IORING_OP_SENDMSG] = {
+ .name = "SENDMSG",
+#if defined(CONFIG_NET)
+ .async_size = sizeof(struct io_async_msghdr),
+ .prep_async = io_sendmsg_prep_async,
+ .cleanup = io_sendmsg_recvmsg_cleanup,
+ .fail = io_sendrecv_fail,
+#endif
+ },
+ [IORING_OP_RECVMSG] = {
+ .name = "RECVMSG",
+#if defined(CONFIG_NET)
+ .async_size = sizeof(struct io_async_msghdr),
+ .prep_async = io_recvmsg_prep_async,
+ .cleanup = io_sendmsg_recvmsg_cleanup,
+ .fail = io_sendrecv_fail,
+#endif
+ },
+ [IORING_OP_TIMEOUT] = {
+ .async_size = sizeof(struct io_timeout_data),
+ .name = "TIMEOUT",
+ },
+ [IORING_OP_TIMEOUT_REMOVE] = {
+ .name = "TIMEOUT_REMOVE",
+ },
+ [IORING_OP_ACCEPT] = {
+ .name = "ACCEPT",
+ },
+ [IORING_OP_ASYNC_CANCEL] = {
+ .name = "ASYNC_CANCEL",
+ },
+ [IORING_OP_LINK_TIMEOUT] = {
+ .async_size = sizeof(struct io_timeout_data),
+ .name = "LINK_TIMEOUT",
+ },
+ [IORING_OP_CONNECT] = {
+ .name = "CONNECT",
+#if defined(CONFIG_NET)
+ .async_size = sizeof(struct io_async_connect),
+ .prep_async = io_connect_prep_async,
+#endif
+ },
+ [IORING_OP_FALLOCATE] = {
+ .name = "FALLOCATE",
+ },
+ [IORING_OP_OPENAT] = {
+ .name = "OPENAT",
+ .cleanup = io_open_cleanup,
+ },
+ [IORING_OP_CLOSE] = {
+ .name = "CLOSE",
+ },
+ [IORING_OP_FILES_UPDATE] = {
+ .name = "FILES_UPDATE",
+ },
+ [IORING_OP_STATX] = {
+ .name = "STATX",
+ .cleanup = io_statx_cleanup,
+ },
+ [IORING_OP_READ] = {
+ .async_size = sizeof(struct io_async_rw),
+ .name = "READ",
+ .fail = io_rw_fail,
+ },
+ [IORING_OP_WRITE] = {
+ .async_size = sizeof(struct io_async_rw),
+ .name = "WRITE",
+ .fail = io_rw_fail,
+ },
+ [IORING_OP_FADVISE] = {
+ .name = "FADVISE",
+ },
+ [IORING_OP_MADVISE] = {
+ .name = "MADVISE",
+ },
+ [IORING_OP_SEND] = {
+ .name = "SEND",
+#if defined(CONFIG_NET)
+ .async_size = sizeof(struct io_async_msghdr),
+ .fail = io_sendrecv_fail,
+ .prep_async = io_send_prep_async,
+#endif
+ },
+ [IORING_OP_RECV] = {
+ .name = "RECV",
+#if defined(CONFIG_NET)
+ .fail = io_sendrecv_fail,
+#endif
+ },
+ [IORING_OP_OPENAT2] = {
+ .name = "OPENAT2",
+ .cleanup = io_open_cleanup,
+ },
+ [IORING_OP_EPOLL_CTL] = {
+ .name = "EPOLL",
+ },
+ [IORING_OP_SPLICE] = {
+ .name = "SPLICE",
+ },
+ [IORING_OP_PROVIDE_BUFFERS] = {
+ .name = "PROVIDE_BUFFERS",
+ },
+ [IORING_OP_REMOVE_BUFFERS] = {
+ .name = "REMOVE_BUFFERS",
+ },
+ [IORING_OP_TEE] = {
+ .name = "TEE",
+ },
+ [IORING_OP_SHUTDOWN] = {
+ .name = "SHUTDOWN",
+ },
+ [IORING_OP_RENAMEAT] = {
+ .name = "RENAMEAT",
+ .cleanup = io_renameat_cleanup,
+ },
+ [IORING_OP_UNLINKAT] = {
+ .name = "UNLINKAT",
+ .cleanup = io_unlinkat_cleanup,
+ },
+ [IORING_OP_MKDIRAT] = {
+ .name = "MKDIRAT",
+ .cleanup = io_mkdirat_cleanup,
+ },
+ [IORING_OP_SYMLINKAT] = {
+ .name = "SYMLINKAT",
+ .cleanup = io_link_cleanup,
+ },
+ [IORING_OP_LINKAT] = {
+ .name = "LINKAT",
+ .cleanup = io_link_cleanup,
+ },
+ [IORING_OP_MSG_RING] = {
+ .name = "MSG_RING",
+ .cleanup = io_msg_ring_cleanup,
+ },
+ [IORING_OP_FSETXATTR] = {
+ .name = "FSETXATTR",
+ .cleanup = io_xattr_cleanup,
+ },
+ [IORING_OP_SETXATTR] = {
+ .name = "SETXATTR",
+ .cleanup = io_xattr_cleanup,
+ },
+ [IORING_OP_FGETXATTR] = {
+ .name = "FGETXATTR",
+ .cleanup = io_xattr_cleanup,
+ },
+ [IORING_OP_GETXATTR] = {
+ .name = "GETXATTR",
+ .cleanup = io_xattr_cleanup,
+ },
+ [IORING_OP_SOCKET] = {
+ .name = "SOCKET",
+ },
+ [IORING_OP_URING_CMD] = {
+ .name = "URING_CMD",
+ .async_size = uring_cmd_pdu_size(1),
+ .prep_async = io_uring_cmd_prep_async,
+ },
+ [IORING_OP_SEND_ZC] = {
+ .name = "SEND_ZC",
+#if defined(CONFIG_NET)
+ .async_size = sizeof(struct io_async_msghdr),
+ .prep_async = io_send_prep_async,
+ .cleanup = io_send_zc_cleanup,
+ .fail = io_sendrecv_fail,
+#endif
+ },
+ [IORING_OP_SENDMSG_ZC] = {
+ .name = "SENDMSG_ZC",
+#if defined(CONFIG_NET)
+ .async_size = sizeof(struct io_async_msghdr),
.prep_async = io_sendmsg_prep_async,
.cleanup = io_send_zc_cleanup,
.fail = io_sendrecv_fail,
-#else
- .prep = io_eopnotsupp_prep,
#endif
},
};
@@ -536,7 +652,7 @@ const struct io_issue_def io_issue_defs[] = {
const char *io_uring_get_opcode(u8 opcode)
{
if (opcode < IORING_OP_LAST)
- return io_issue_defs[opcode].name;
+ return io_cold_defs[opcode].name;
return "INVALID";
}
@@ -544,12 +660,13 @@ void __init io_uring_optable_init(void)
{
int i;
+ BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
BUG_ON(!io_issue_defs[i].prep);
if (io_issue_defs[i].prep != io_eopnotsupp_prep)
BUG_ON(!io_issue_defs[i].issue);
- WARN_ON_ONCE(!io_issue_defs[i].name);
+ WARN_ON_ONCE(!io_cold_defs[i].name);
}
}
diff --git a/io_uring/opdef.h b/io_uring/opdef.h
index d718e2ab1ff7..c22c8696e749 100644
--- a/io_uring/opdef.h
+++ b/io_uring/opdef.h
@@ -29,19 +29,24 @@ struct io_issue_def {
unsigned iopoll_queue : 1;
/* opcode specific path will handle ->async_data allocation if needed */
unsigned manual_alloc : 1;
+
+ int (*issue)(struct io_kiocb *, unsigned int);
+ int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);
+};
+
+struct io_cold_def {
/* size of async data needed, if any */
unsigned short async_size;
const char *name;
- int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);
- int (*issue)(struct io_kiocb *, unsigned int);
int (*prep_async)(struct io_kiocb *);
void (*cleanup)(struct io_kiocb *);
void (*fail)(struct io_kiocb *);
};
extern const struct io_issue_def io_issue_defs[];
+extern const struct io_cold_def io_cold_defs[];
void io_uring_optable_init(void);
#endif
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 54b44b9b736c..a8a2eb7ee27a 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -516,7 +516,7 @@ static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
struct io_rw_state *s, bool force)
{
- if (!force && !io_issue_defs[req->opcode].prep_async)
+ if (!force && !io_cold_defs[req->opcode].prep_async)
return 0;
if (!req_has_async_data(req)) {
struct io_async_rw *iorw;
