[PATCH 5/5] io_uring: sendmsg/recvmsg with registered buffers

[Pavel Begunkov <asml.silence@xxxxxxxxx>]

Add support of registered buffers to sendmsg() and recvmsg(). As with
previous one, it uses IO_MSG_FIXED, last bit of flags, which is cleared
before going into net stack.

Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
 fs/io_uring.c | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 390495170fb0..7b13dafc84ab 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -4488,7 +4488,7 @@ static int io_setup_async_msg(struct io_kiocb *req,
 	memcpy(async_msg, kmsg, sizeof(*kmsg));
 	async_msg->msg.msg_name = &async_msg->addr;
 	/* if iov is not set, it uses fast_iov */
-	if (!async_msg->iov)
+	if (!async_msg->iov && !(req->sr_msg.msg_flags & IO_MSG_FIXED))
 		async_msg->msg.msg_iter.iov = async_msg->fast_iov;
 	return -EAGAIN;
 }
@@ -4508,7 +4508,8 @@ static int __io_msg_copy_hdr(struct io_kiocb *req,
 	if (ret)
 		return ret;
 
-	if (req->flags & REQ_F_BUFFER_SELECT) {
+	if ((req->flags & REQ_F_BUFFER_SELECT) ||
+	    (sr->msg_flags & IO_MSG_FIXED)) {
 		if (iov_len > 1)
 			return -EINVAL;
 		if (copy_from_user(iomsg->iov, uiov, sizeof(*uiov)))
@@ -4548,7 +4549,8 @@ static int __io_compat_msg_copy_hdr(struct io_kiocb *req,
 		return ret;
 
 	uiov = compat_ptr(ptr);
-	if (req->flags & REQ_F_BUFFER_SELECT) {
+	if ((req->flags & REQ_F_BUFFER_SELECT) ||
+	    (sr->msg_flags & IO_MSG_FIXED)) {
 		compat_ssize_t clen;
 
 		if (len > 1)
@@ -4591,6 +4593,7 @@ static struct io_buffer *io_recv_buffer_select(struct io_kiocb *req,
 static int io_import_msg(struct io_kiocb *req, struct io_async_msghdr *iomsg,
 			 int rw)
 {
+	struct io_sr_msg *sr = &req->sr_msg;
 	struct io_buffer *kbuf;
 	int ret;
 
@@ -4606,7 +4609,7 @@ static int io_import_msg(struct io_kiocb *req, struct io_async_msghdr *iomsg,
 		return ret;
 
 	if (req->flags & REQ_F_BUFFER_SELECT) {
-		if (rw != READ)
+		if (rw != READ || (sr->msg_flags & IO_MSG_FIXED))
 			return -EINVAL;
 		/* init is always done with uring_lock held */
 		kbuf = io_recv_buffer_select(req, false);
@@ -4614,7 +4617,14 @@ static int io_import_msg(struct io_kiocb *req, struct io_async_msghdr *iomsg,
 			return PTR_ERR(kbuf);
 		iomsg->fast_iov[0].iov_base = u64_to_user_ptr(kbuf->addr);
 		iov_iter_init(&iomsg->msg.msg_iter, READ, iomsg->fast_iov, 1,
-			      req->sr_msg.len);
+			      sr->len);
+	} else if (sr->msg_flags & IO_MSG_FIXED) {
+		struct iovec *iov = &iomsg->fast_iov[0];
+
+		ret = io_import_fixed(req, rw, (u64)iov->iov_base, iov->iov_len,
+				      &iomsg->msg.msg_iter);
+		if (ret < 0)
+			return ret;
 	}
 	return 0;
 }
@@ -4631,6 +4641,7 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	sr->msg_flags = READ_ONCE(sqe->msg_flags);
 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
 	sr->len = READ_ONCE(sqe->len);
+	req->buf_index = READ_ONCE(sqe->buf_index);
 
 #ifdef CONFIG_COMPAT
 	if (req->ctx->compat)
@@ -4665,7 +4676,7 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock,
 		kmsg = &iomsg;
 	}
 
-	flags = req->sr_msg.msg_flags;
+	flags = req->sr_msg.msg_flags & ~IO_MSG_FIXED;
 	if (flags & MSG_DONTWAIT)
 		req->flags |= REQ_F_NOWAIT;
 	else if (force_nonblock)
@@ -4754,6 +4765,7 @@ static int io_recvmsg_prep(struct io_kiocb *req,
 	sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
 	sr->len = READ_ONCE(sqe->len);
 	sr->bgid = READ_ONCE(sqe->buf_group);
+	req->buf_index = READ_ONCE(sqe->buf_index);
 
 #ifdef CONFIG_COMPAT
 	if (req->ctx->compat)
@@ -4788,7 +4800,7 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock,
 		kmsg = &iomsg;
 	}
 
-	flags = req->sr_msg.msg_flags;
+	flags = req->sr_msg.msg_flags & ~IO_MSG_FIXED;
 	if (flags & MSG_DONTWAIT)
 		req->flags |= REQ_F_NOWAIT;
 	else if (force_nonblock)
-- 
2.24.0