This works very much like the receive side, except for sends. The idea
is that an application can fill outgoing buffers in a provided buffer
group, and then arm a single send that will service them all. For now
this variant just terminates when we are out of buffers to send, and
hence the application needs to re-arm it if IORING_CQE_F_MORE isn't
set, as per usual for multishot requests.
This only enables it for IORING_OP_SEND, IORING_OP_SENDMSG is coming
in a separate patch. However, this patch does do a lot of the prep
work that makes wiring up the sendmsg variant pretty trivial. They
share the prep side.
Enabling multishot for sends is, again, identical to the receive side.
The app sets IORING_SEND_MULTISHOT in sqe->ioprio. This flag is also
the same as IORING_RECV_MULTISHOT.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
include/uapi/linux/io_uring.h | 8 +++
io_uring/net.c | 98 +++++++++++++++++++++++++++++------
2 files changed, 89 insertions(+), 17 deletions(-)
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 74c3afac9c63..6766e78ee03b 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -351,9 +351,17 @@ enum io_uring_op {
* 0 is reported if zerocopy was actually possible.
* IORING_NOTIF_USAGE_ZC_COPIED if data was copied
* (at least partially).
+ *
+ * IORING_SEND_MULTISHOT Multishot send. Like the recv equivalent, must
+ * be used with provided buffers. Keeps sending
+ * from the given buffer group ID until it is
+ * empty. Sets IORING_CQE_F_MORE if more
+ * completions should be expected on behalf of
+ * the same SQE.
*/
#define IORING_RECVSEND_POLL_FIRST (1U << 0)
#define IORING_RECV_MULTISHOT (1U << 1)
+#define IORING_SEND_MULTISHOT IORING_RECV_MULTISHOT
#define IORING_RECVSEND_FIXED_BUF (1U << 2)
#define IORING_SEND_ZC_REPORT_USAGE (1U << 3)
diff --git a/io_uring/net.c b/io_uring/net.c
index 30afb394efd7..8237ac5c957f 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -204,6 +204,16 @@ static int io_setup_async_msg(struct io_kiocb *req,
return -EAGAIN;
}
+static inline void io_mshot_prep_retry(struct io_kiocb *req)
+{
+ struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
+
+ req->flags &= ~REQ_F_BL_EMPTY;
+ sr->done_io = 0;
+ sr->len = 0; /* get from the provided buffer */
+ req->buf_index = sr->buf_group;
+}
+
static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg)
{
int hdr;
@@ -401,6 +411,8 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
kfree(io->free_iov);
}
+#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_SEND_MULTISHOT)
+
int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
@@ -417,11 +429,19 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
sr->len = READ_ONCE(sqe->len);
sr->flags = READ_ONCE(sqe->ioprio);
- if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
+ if (sr->flags & ~SENDMSG_FLAGS)
return -EINVAL;
sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
if (sr->msg_flags & MSG_DONTWAIT)
req->flags |= REQ_F_NOWAIT;
+ if (sr->flags & IORING_SEND_MULTISHOT) {
+ if (!(req->flags & REQ_F_BUFFER_SELECT))
+ return -EINVAL;
+ if (sr->msg_flags & MSG_WAITALL)
+ return -EINVAL;
+ req->flags |= REQ_F_APOLL_MULTISHOT;
+ sr->buf_group = req->buf_index;
+ }
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
@@ -431,6 +451,44 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0;
}
+static inline bool io_send_finish(struct io_kiocb *req, int *ret,
+ struct msghdr *msg, unsigned issue_flags)
+{
+ bool mshot_finished = *ret <= 0;
+ unsigned int cflags;
+
+ cflags = io_put_kbuf(req, issue_flags);
+
+ if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
+ io_req_set_res(req, *ret, cflags);
+ *ret = IOU_OK;
+ return true;
+ }
+
+ if (mshot_finished || req->flags & REQ_F_BL_EMPTY)
+ goto finish;
+
+ /*
+ * Fill CQE for this receive and see if we should keep trying to
+ * receive from this socket.
+ */
+ if (io_fill_cqe_req_aux(req, issue_flags & IO_URING_F_COMPLETE_DEFER,
+ *ret, cflags | IORING_CQE_F_MORE)) {
+ io_mshot_prep_retry(req);
+ *ret = IOU_ISSUE_SKIP_COMPLETE;
+ return false;
+ }
+
+ /* Otherwise stop multishot but use the current result. */
+finish:
+ io_req_set_res(req, *ret, cflags);
+ if (issue_flags & IO_URING_F_MULTISHOT)
+ *ret = IOU_STOP_MULTISHOT;
+ else
+ *ret = IOU_OK;
+ return true;
+}
+
int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
@@ -511,7 +569,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
struct sockaddr_storage __address;
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
size_t len = sr->len;
- unsigned int cflags;
struct socket *sock;
struct msghdr msg;
unsigned flags;
@@ -542,10 +599,14 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
(sr->flags & IORING_RECVSEND_POLL_FIRST))
return io_setup_async_addr(req, &__address, issue_flags);
+ if (!io_check_multishot(req, issue_flags))
+ return -EAGAIN;
+
sock = sock_from_file(req->file);
if (unlikely(!sock))
return -ENOTSOCK;
+retry_multishot:
if (io_do_buffer_select(req)) {
void __user *buf;
@@ -570,8 +631,16 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
msg.msg_flags = flags;
ret = sock_sendmsg(sock, &msg);
if (ret < min_ret) {
- if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
- return io_setup_async_addr(req, &__address, issue_flags);
+ if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) {
+ ret = io_setup_async_addr(req, &__address, issue_flags);
+ if (ret != -EAGAIN)
+ return ret;
+ if (issue_flags & IO_URING_F_MULTISHOT) {
+ io_kbuf_recycle(req, issue_flags);
+ return IOU_ISSUE_SKIP_COMPLETE;
+ }
+ return -EAGAIN;
+ }
if (ret > 0 && io_net_retry(sock, flags)) {
sr->len -= ret;
@@ -588,9 +657,13 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
ret += sr->done_io;
else if (sr->done_io)
ret = sr->done_io;
- cflags = io_put_kbuf(req, issue_flags);
- io_req_set_res(req, ret, cflags);
- return IOU_OK;
+ else
+ io_kbuf_recycle(req, issue_flags);
+
+ if (!io_send_finish(req, &ret, &msg, issue_flags))
+ goto retry_multishot;
+
+ return ret;
}
int io_recvmsg_prep_async(struct io_kiocb *req)
@@ -654,15 +727,6 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0;
}
-static inline void io_recv_prep_retry(struct io_kiocb *req)
-{
- struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
-
- sr->done_io = 0;
- sr->len = 0; /* get from the provided buffer */
- req->buf_index = sr->buf_group;
-}
-
/*
* Finishes io_recv and io_recvmsg.
*
@@ -697,7 +761,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
int mshot_retry_ret = IOU_ISSUE_SKIP_COMPLETE;
- io_recv_prep_retry(req);
+ io_mshot_prep_retry(req);
/* Known not-empty or unknown state, retry */
if (cflags & IORING_CQE_F_SOCK_NONEMPTY || msg->msg_inq == -1) {
if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY)
--
2.43.0
