This works very much like the receive side, except for sends. The idea
is that an application can fill outgoing buffers in a provided buffer
group, and then arm a single send that will service them all. For now
this variant just terminates when we are out of buffers to send, and
hence the application needs to re-arm it if IORING_CQE_F_MORE isn't
set, as per usual for multishot requests.
This only enables it for IORING_OP_SEND, IORING_OP_SENDMSG is coming
in a separate patch. However, this patch does do a lot of the prep
work that makes wiring up the sendmsg variant pretty trivial. They
share the prep side.
Enabling multishot for sends is, again, identical to the receive side.
The app sets IORING_SEND_MULTISHOT in sqe->ioprio. This flag is also
the same as IORING_RECV_MULTISHOT.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
include/uapi/linux/io_uring.h | 8 ++++
io_uring/net.c | 90 ++++++++++++++++++++++++++++++-----
2 files changed, 86 insertions(+), 12 deletions(-)
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 061147cdfbb0..feacc64c90a0 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -351,9 +351,17 @@ enum io_uring_op {
* 0 is reported if zerocopy was actually possible.
* IORING_NOTIF_USAGE_ZC_COPIED if data was copied
* (at least partially).
+ *
+ * IORING_SEND_MULTISHOT Multishot send. Like the recv equivalent, must
+ * be used with provided buffers. Keeps sending
+ * from the given buffer group ID until it is
+ * empty. Sets IORING_CQE_F_MORE if more
+ * completions should be expected on behalf of
+ * the same SQE.
*/
#define IORING_RECVSEND_POLL_FIRST (1U << 0)
#define IORING_RECV_MULTISHOT (1U << 1)
+#define IORING_SEND_MULTISHOT IORING_RECV_MULTISHOT
#define IORING_RECVSEND_FIXED_BUF (1U << 2)
#define IORING_SEND_ZC_REPORT_USAGE (1U << 3)
diff --git a/io_uring/net.c b/io_uring/net.c
index 9ef11883a34a..0b990df04ac7 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -411,6 +411,8 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
kfree(io->free_iov);
}
+#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_SEND_MULTISHOT)
+
int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
@@ -427,11 +429,17 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
sr->len = READ_ONCE(sqe->len);
sr->flags = READ_ONCE(sqe->ioprio);
- if (sr->flags & ~IORING_RECVSEND_POLL_FIRST)
+ if (sr->flags & ~SENDMSG_FLAGS)
return -EINVAL;
sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
if (sr->msg_flags & MSG_DONTWAIT)
req->flags |= REQ_F_NOWAIT;
+ if (sr->flags & IORING_SEND_MULTISHOT) {
+ if (!(req->flags & REQ_F_BUFFER_SELECT))
+ return -EINVAL;
+ req->flags |= REQ_F_APOLL_MULTISHOT;
+ sr->buf_group = req->buf_index;
+ }
#ifdef CONFIG_COMPAT
if (req->ctx->compat)
@@ -441,6 +449,44 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return 0;
}
+static inline bool io_send_finish(struct io_kiocb *req, int *ret,
+ struct msghdr *msg, unsigned issue_flags)
+{
+ bool mshot_finished = *ret <= 0;
+ unsigned int cflags;
+
+ cflags = io_put_kbuf(req, issue_flags);
+
+ if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
+ io_req_set_res(req, *ret, cflags);
+ *ret = IOU_OK;
+ return true;
+ }
+
+ if (mshot_finished || req->flags & REQ_F_BL_EMPTY)
+ goto finish;
+
+ /*
+ * Fill CQE for this receive and see if we should keep trying to
+ * receive from this socket.
+ */
+ if (io_fill_cqe_req_aux(req, issue_flags & IO_URING_F_COMPLETE_DEFER,
+ *ret, cflags | IORING_CQE_F_MORE)) {
+ io_mshot_prep_retry(req);
+ *ret = IOU_ISSUE_SKIP_COMPLETE;
+ return false;
+ }
+
+ /* Otherwise stop multishot but use the current result. */
+finish:
+ io_req_set_res(req, *ret, cflags);
+ if (issue_flags & IO_URING_F_MULTISHOT)
+ *ret = IOU_STOP_MULTISHOT;
+ else
+ *ret = IOU_OK;
+ return true;
+}
+
int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
@@ -523,7 +569,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
size_t len = sr->len;
struct socket *sock;
struct msghdr msg;
- unsigned int cflags;
unsigned flags;
int min_ret = 0;
int ret;
@@ -552,10 +597,18 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
(sr->flags & IORING_RECVSEND_POLL_FIRST))
return io_setup_async_addr(req, &__address, issue_flags);
+ if (!io_check_multishot(req, issue_flags))
+ return -EAGAIN;
+
sock = sock_from_file(req->file);
if (unlikely(!sock))
return -ENOTSOCK;
+ flags = sr->msg_flags;
+ if (issue_flags & IO_URING_F_NONBLOCK)
+ flags |= MSG_DONTWAIT;
+
+retry_multishot:
if (io_do_buffer_select(req)) {
void __user *buf;
@@ -570,19 +623,28 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(ret))
return ret;
- flags = sr->msg_flags;
- if (issue_flags & IO_URING_F_NONBLOCK)
- flags |= MSG_DONTWAIT;
- if (flags & MSG_WAITALL)
+ /*
+ * If MSG_WAITALL is set, or this is a multishot send, then we need
+ * the full amount. If just multishot is set, if we do a short send
+ * then we complete the multishot sequence rather than continue on.
+ */
+ if (flags & MSG_WAITALL || req->flags & REQ_F_APOLL_MULTISHOT)
min_ret = iov_iter_count(&msg.msg_iter);
flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
msg.msg_flags = flags;
ret = sock_sendmsg(sock, &msg);
if (ret < min_ret) {
- if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
- return io_setup_async_addr(req, &__address, issue_flags);
-
+ if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) {
+ ret = io_setup_async_addr(req, &__address, issue_flags);
+ if (ret != -EAGAIN)
+ return ret;
+ if (issue_flags & IO_URING_F_MULTISHOT) {
+ io_kbuf_recycle(req, issue_flags);
+ return IOU_ISSUE_SKIP_COMPLETE;
+ }
+ return -EAGAIN;
+ }
if (ret > 0 && io_net_retry(sock, flags)) {
sr->len -= ret;
sr->buf += ret;
@@ -598,9 +660,13 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags)
ret += sr->done_io;
else if (sr->done_io)
ret = sr->done_io;
- cflags = io_put_kbuf(req, issue_flags);
- io_req_set_res(req, ret, cflags);
- return IOU_OK;
+ else
+ io_kbuf_recycle(req, issue_flags);
+
+ if (!io_send_finish(req, &ret, &msg, issue_flags))
+ goto retry_multishot;
+
+ return ret;
}
int io_recvmsg_prep_async(struct io_kiocb *req)
--
2.43.0
