Prepare for supporting kernel buffer in case of io group, in which group
leader leases kernel buffer to io_uring, and consumed by io_uring OPs.
So reuse io_mapped_buf for group kernel buffer, and unfortunately
io_import_fixed() can't be reused since userspace fixed buffer is
virt-contiguous, but it isn't true for kernel buffer.
Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx>
---
include/linux/io_uring_types.h | 19 +++++++++++++++++++
io_uring/kbuf.c | 34 ++++++++++++++++++++++++++++++++++
io_uring/kbuf.h | 3 +++
io_uring/rsrc.c | 1 +
io_uring/rsrc.h | 10 ----------
5 files changed, 57 insertions(+), 10 deletions(-)
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index d060ce5e6145..03abaeef4a67 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -2,6 +2,7 @@
#define IO_URING_TYPES_H
#include <linux/blkdev.h>
+#include <linux/bvec.h>
#include <linux/hashtable.h>
#include <linux/task_work.h>
#include <linux/bitmap.h>
@@ -39,6 +40,24 @@ enum io_uring_cmd_flags {
IO_URING_F_COMPAT = (1 << 12),
};
+struct io_mapped_buf {
+ u64 addr;
+ unsigned int len;
+ unsigned int nr_bvecs;
+ refcount_t refs;
+ union {
+ /* for userspace buffer only */
+ unsigned int acct_pages;
+ /* offset in the 1st bvec, for kbuf only */
+ unsigned int offset;
+ };
+ const struct bio_vec *pbvec; /* pbvec is only for kbuf */
+ unsigned int folio_shift:6;
+ unsigned int dir:1; /* ITER_DEST or ITER_SOURCE */
+ unsigned int kbuf:1; /* kernel buffer or not */
+ struct bio_vec bvec[] __counted_by(nr_bvecs);
+};
+
struct io_wq_work_node {
struct io_wq_work_node *next;
};
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index d407576ddfb7..c4a776860cb4 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -838,3 +838,37 @@ int io_pbuf_mmap(struct file *file, struct vm_area_struct *vma)
io_put_bl(ctx, bl);
return ret;
}
+
+/*
+ * kernel buffer is built over generic bvec, and can't be always
+ * virt-contiguous, which is different with userspace fixed buffer,
+ * so we can't reuse io_import_fixed() here
+ *
+ * Also kernel buffer lifetime is bound with request, and we needn't
+ * to use rsrc_node to track its lifetime
+ */
+int io_import_kbuf(int ddir, struct iov_iter *iter,
+ const struct io_mapped_buf *kbuf,
+ u64 buf_off, size_t len)
+{
+ unsigned long offset = kbuf->offset;
+
+ WARN_ON_ONCE(!kbuf->kbuf);
+
+ if (ddir != kbuf->dir)
+ return -EINVAL;
+
+ if (unlikely(buf_off > kbuf->len))
+ return -EFAULT;
+
+ if (unlikely(len > kbuf->len - buf_off))
+ return -EFAULT;
+
+ offset += buf_off;
+ iov_iter_bvec(iter, ddir, kbuf->pbvec, kbuf->nr_bvecs, offset + len);
+
+ if (offset)
+ iov_iter_advance(iter, offset);
+
+ return 0;
+}
diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index 36aadfe5ac00..04ccd52dd0ad 100644
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -88,6 +88,9 @@ void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl);
struct io_buffer_list *io_pbuf_get_bl(struct io_ring_ctx *ctx,
unsigned long bgid);
int io_pbuf_mmap(struct file *file, struct vm_area_struct *vma);
+int io_import_kbuf(int ddir, struct iov_iter *iter,
+ const struct io_mapped_buf *kbuf,
+ u64 buf_off, size_t len);
static inline bool io_kbuf_recycle_ring(struct io_kiocb *req)
{
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 99ff2797e6ec..b0b60ae0456a 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -771,6 +771,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
imu->len = iov->iov_len;
imu->nr_bvecs = nr_pages;
imu->folio_shift = PAGE_SHIFT;
+ imu->kbuf = 0;
if (coalesced)
imu->folio_shift = data.folio_shift;
refcount_set(&imu->refs, 1);
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index bf0824b4beb6..3bc3a484fbba 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -28,16 +28,6 @@ struct io_rsrc_node {
};
};
-struct io_mapped_buf {
- u64 addr;
- unsigned int len;
- unsigned int nr_bvecs;
- refcount_t refs;
- unsigned int acct_pages;
- unsigned int folio_shift:6;
- struct bio_vec bvec[] __counted_by(nr_bvecs);
-};
-
struct io_imu_folio_data {
/* Head folio can be partially included in the fixed buf */
unsigned int nr_pages_head;
--
2.47.0
