From: Jens Axboe <axboe@xxxxxxxxx>
io_uring treats all completions the same - they post a completion event,
or more, and anyone waiting on event completions will see each event as
it gets posted.
However, some events may be more interesting that others. For a request
and response type model, it's not uncommon to have send/write events
that are submitted with a recv/read type of request. While the app does
want to see a successful send/write completion eventually, it need not
handle it upfront as it would want to do with a recv/read, as it isn't
time sensitive. Generally, a send/write completion will just mean that
a buffer can get recycled/reused, whereas a recv/read completion needs
acting upon (and a response sent).
This can be somewhat tricky to handle if many requests and responses
are being handled, and the app generally needs to track the number of
pending sends/writes to be able to sanely wait on just new incoming
recv/read requests. And even with that, an application would still
like to see a completion for a short/failed send/write immediately.
Add infrastructure to account inline completions, such that they can
be deducted from the 'wait_nr' being passed in via a submit_and_wait()
type of situation. Inline completions are ones that complete directly
inline from submission, such as a send to a socket where there's
enough space to accomodate the data being sent.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
[pavel: rebased onto iosets]
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
include/linux/io_uring_types.h | 1 +
include/uapi/linux/io_uring.h | 4 ++++
io_uring/io_uring.c | 12 ++++++++++--
io_uring/register.c | 2 +-
4 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 79f38c07642d..f04444f9356a 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -213,6 +213,7 @@ struct io_submit_state {
bool need_plug;
bool cq_flush;
unsigned short submit_nr;
+ unsigned short inline_completions;
struct blk_plug plug;
};
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 6a432383e7c3..e6d10fba8ae2 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -899,6 +899,10 @@ struct io_uring_recvmsg_out {
__u32 flags;
};
+enum {
+ IOSQE_SET_F_HINT_IGNORE_INLINE = 1,
+};
+
struct io_uring_ioset_reg {
__u64 flags;
__u64 __resv[3];
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index cf688a9ff737..6e89435c243d 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1575,6 +1575,9 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
struct io_kiocb *req = container_of(node, struct io_kiocb,
comp_list);
+ if (req->ioset->flags & IOSQE_SET_F_HINT_IGNORE_INLINE)
+ state->inline_completions++;
+
if (unlikely(req->flags & (REQ_F_CQE_SKIP | REQ_F_GROUP))) {
if (req->flags & REQ_F_GROUP) {
io_complete_group_req(req);
@@ -2511,6 +2514,7 @@ static void io_submit_state_start(struct io_submit_state *state,
state->plug_started = false;
state->need_plug = max_ios > 2;
state->submit_nr = max_ios;
+ state->inline_completions = 0;
/* set only head, no need to init link_last in advance */
state->link.head = NULL;
state->group.head = NULL;
@@ -3611,6 +3615,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
size_t, argsz)
{
struct io_ring_ctx *ctx;
+ int inline_complete = 0;
struct file *file;
long ret;
@@ -3676,6 +3681,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
mutex_unlock(&ctx->uring_lock);
goto out;
}
+ inline_complete = ctx->submit_state.inline_completions;
if (flags & IORING_ENTER_GETEVENTS) {
if (ctx->syscall_iopoll)
goto iopoll_locked;
@@ -3713,8 +3719,10 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
ret2 = io_get_ext_arg(ctx, flags, argp, &ext_arg);
if (likely(!ret2)) {
- min_complete = min(min_complete,
- ctx->cq_entries);
+ if (min_complete > ctx->cq_entries)
+ min_complete = ctx->cq_entries;
+ else
+ min_complete += inline_complete;
ret2 = io_cqring_wait(ctx, min_complete, flags,
&ext_arg);
}
diff --git a/io_uring/register.c b/io_uring/register.c
index e7571dc46da5..f87ec7b773bd 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -92,7 +92,7 @@ static int io_update_ioset(struct io_ring_ctx *ctx,
{
if (!(ctx->flags & IORING_SETUP_IOSET))
return -EINVAL;
- if (reg->flags)
+ if (reg->flags & ~IOSQE_SET_F_HINT_IGNORE_INLINE)
return -EINVAL;
if (reg->__resv[0] || reg->__resv[1] || reg->__resv[2])
return -EINVAL;
--
2.46.0
