Add ITER_BVEC_FLAG_FIXED iov iter flag, which will allow us to reuse
passed in bvec instead of copying it. In particular it means that
iter->bvec won't be freed and page references are taken remain so
until callees don't need them, including asynchronous execution.
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
fs/io_uring.c | 1 +
include/linux/uio.h | 14 +++++++++++---
2 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index c536462920a3..9ff2805d0075 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2920,6 +2920,7 @@ static ssize_t io_import_fixed(struct io_kiocb *req, int rw,
}
}
+ iter->type |= ITER_BVEC_FLAG_FIXED;
return len;
}
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 72d88566694e..af626eb970cf 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -18,6 +18,8 @@ struct kvec {
};
enum iter_type {
+ ITER_BVEC_FLAG_FIXED = 2,
+
/* iter types */
ITER_IOVEC = 4,
ITER_KVEC = 8,
@@ -29,8 +31,9 @@ enum iter_type {
struct iov_iter {
/*
* Bit 0 is the read/write bit, set if we're writing.
- * Bit 1 is the BVEC_FLAG_NO_REF bit, set if type is a bvec and
- * the caller isn't expecting to drop a page reference when done.
+ * Bit 1 is the BVEC_FLAG_FIXED bit, set if type is a bvec and the
+ * caller ensures that page references and memory baking bvec won't
+ * go away until callees finish with them.
*/
unsigned int type;
size_t iov_offset;
@@ -52,7 +55,7 @@ struct iov_iter {
static inline enum iter_type iov_iter_type(const struct iov_iter *i)
{
- return i->type & ~(READ | WRITE);
+ return i->type & ~(READ | WRITE | ITER_BVEC_FLAG_FIXED);
}
static inline bool iter_is_iovec(const struct iov_iter *i)
@@ -85,6 +88,11 @@ static inline unsigned char iov_iter_rw(const struct iov_iter *i)
return i->type & (READ | WRITE);
}
+static inline unsigned char iov_iter_bvec_fixed(const struct iov_iter *i)
+{
+ return i->type & ITER_BVEC_FLAG_FIXED;
+}
+
/*
* Total number of bytes covered by an iovec.
*
--
2.24.0
