From: Hou Tao <houtao1@xxxxxxxxxx>
When reading a file kept in virtiofs from kernel (e.g., insmod a kernel
module), if the cache of virtiofs is disabled, the read buffer will be
passed to virtiofs through out_args[0].value instead of pages. Because
virtiofs can't get the pages for the read buffer, virtio_fs_argbuf_new()
will create a bounce buffer for the read buffer by using kmalloc() and
copy the read buffer into bounce buffer. If the read buffer is large
(e.g., 1MB), the allocation will incur significant stress on the memory
subsystem.
So instead of allocating bounce buffer by using kmalloc(), allocate a
bounce buffer which is backed by scattered pages. The original idea is
to use vmap(), but the use of GFP_ATOMIC is no possible for vmap(). To
simplify the copy operations in the bounce buffer, use a bio_vec flex
array to represent the argbuf. Also add an is_flat field in struct
virtio_fs_argbuf to distinguish between kmalloc-ed and scattered bounce
buffer.
Signed-off-by: Hou Tao <houtao1@xxxxxxxxxx>
---
fs/fuse/virtio_fs.c | 163 ++++++++++++++++++++++++++++++++++++++++----
1 file changed, 149 insertions(+), 14 deletions(-)
diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c
index f10fff7f23a0f..ffea684bd100d 100644
--- a/fs/fuse/virtio_fs.c
+++ b/fs/fuse/virtio_fs.c
@@ -86,10 +86,27 @@ struct virtio_fs_req_work {
struct work_struct done_work;
};
-struct virtio_fs_argbuf {
+struct virtio_fs_flat_argbuf {
DECLARE_FLEX_ARRAY(u8, buf);
};
+struct virtio_fs_scattered_argbuf {
+ unsigned int size;
+ unsigned int nr;
+ DECLARE_FLEX_ARRAY(struct bio_vec, bvec);
+};
+
+struct virtio_fs_argbuf {
+ bool is_flat;
+ /* There is flexible array in the end of these two struct
+ * definitions, so they must be the last field.
+ */
+ union {
+ struct virtio_fs_flat_argbuf f;
+ struct virtio_fs_scattered_argbuf s;
+ };
+};
+
static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
struct fuse_req *req, bool in_flight);
@@ -408,42 +425,143 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work)
}
}
+static unsigned int virtio_fs_argbuf_len(unsigned int in_args_len,
+ unsigned int out_args_len,
+ bool is_flat)
+{
+ if (is_flat)
+ return in_args_len + out_args_len;
+
+ /*
+ * Align in_args_len with PAGE_SIZE to reduce the total number of
+ * sg entries when the value of out_args_len (e.g., the length of
+ * read buffer) is page-aligned.
+ */
+ return round_up(in_args_len, PAGE_SIZE) +
+ round_up(out_args_len, PAGE_SIZE);
+}
+
static void virtio_fs_argbuf_free(struct virtio_fs_argbuf *argbuf)
{
+ unsigned int i;
+
+ if (!argbuf)
+ return;
+
+ if (argbuf->is_flat)
+ goto free_argbuf;
+
+ for (i = 0; i < argbuf->s.nr; i++)
+ __free_page(argbuf->s.bvec[i].bv_page);
+
+free_argbuf:
kfree(argbuf);
}
static struct virtio_fs_argbuf *virtio_fs_argbuf_new(struct fuse_args *args,
- gfp_t gfp)
+ gfp_t gfp, bool is_flat)
{
struct virtio_fs_argbuf *argbuf;
unsigned int numargs;
- unsigned int len;
+ unsigned int in_len, out_len, len;
+ unsigned int i, nr;
numargs = args->in_numargs - args->in_pages;
- len = fuse_len_args(numargs, (struct fuse_arg *) args->in_args);
+ in_len = fuse_len_args(numargs, (struct fuse_arg *) args->in_args);
numargs = args->out_numargs - args->out_pages;
- len += fuse_len_args(numargs, args->out_args);
+ out_len = fuse_len_args(numargs, args->out_args);
+ len = virtio_fs_argbuf_len(in_len, out_len, is_flat);
+
+ if (is_flat) {
+ argbuf = kmalloc(struct_size(argbuf, f.buf, len), gfp);
+ if (argbuf)
+ argbuf->is_flat = true;
+
+ return argbuf;
+ }
+
+ nr = len >> PAGE_SHIFT;
+ argbuf = kmalloc(struct_size(argbuf, s.bvec, nr), gfp);
+ if (!argbuf)
+ return NULL;
+
+ argbuf->is_flat = false;
+ argbuf->s.size = len;
+ argbuf->s.nr = 0;
+ for (i = 0; i < nr; i++) {
+ struct page *page;
+
+ page = alloc_page(gfp);
+ if (!page) {
+ virtio_fs_argbuf_free(argbuf);
+ return NULL;
+ }
+ bvec_set_page(&argbuf->s.bvec[i], page, PAGE_SIZE, 0);
+ argbuf->s.nr++;
+ }
+
+ /* Zero the unused space for in_args */
+ if (in_len & ~PAGE_MASK) {
+ struct iov_iter iter;
+ unsigned int to_zero;
+
+ iov_iter_bvec(&iter, ITER_DEST, argbuf->s.bvec, argbuf->s.nr,
+ argbuf->s.size);
+ iov_iter_advance(&iter, in_len);
- argbuf = kmalloc(struct_size(argbuf, buf, len), gfp);
+ to_zero = PAGE_SIZE - (in_len & ~PAGE_MASK);
+ iov_iter_zero(to_zero, &iter);
+ }
return argbuf;
}
static unsigned int virtio_fs_argbuf_setup_sg(struct virtio_fs_argbuf *argbuf,
unsigned int offset,
- unsigned int len,
+ unsigned int *len,
struct scatterlist *sg)
{
- sg_init_one(sg, argbuf->buf + offset, len);
- return 1;
+ struct bvec_iter bi = {
+ .bi_size = offset + *len,
+ };
+ struct scatterlist *cur;
+ struct bio_vec bv;
+
+ if (argbuf->is_flat) {
+ sg_init_one(sg, argbuf->f.buf + offset, *len);
+ return 1;
+ }
+
+ cur = sg;
+ bvec_iter_advance(argbuf->s.bvec, &bi, offset);
+ for_each_bvec(bv, argbuf->s.bvec, bi, bi) {
+ sg_init_table(cur, 1);
+ sg_set_page(cur, bv.bv_page, bv.bv_len, bv.bv_offset);
+ cur++;
+ }
+ *len = round_up(*len, PAGE_SIZE);
+
+ return cur - sg;
}
static void virtio_fs_argbuf_copy_from_in_arg(struct virtio_fs_argbuf *argbuf,
unsigned int offset,
const void *src, unsigned int len)
{
- memcpy(argbuf->buf + offset, src, len);
+ struct iov_iter iter;
+ unsigned int copied;
+
+ if (argbuf->is_flat) {
+ memcpy(argbuf->f.buf + offset, src, len);
+ return;
+ }
+
+ iov_iter_bvec(&iter, ITER_DEST, argbuf->s.bvec,
+ argbuf->s.nr, argbuf->s.size);
+ iov_iter_advance(&iter, offset);
+
+ copied = _copy_to_iter(src, len, &iter);
+ WARN_ON_ONCE(copied != len);
}
static unsigned int
@@ -451,15 +569,32 @@ virtio_fs_argbuf_out_args_offset(struct virtio_fs_argbuf *argbuf,
const struct fuse_args *args)
{
unsigned int num_in = args->in_numargs - args->in_pages;
+ unsigned int offset = fuse_len_args(num_in,
+ (struct fuse_arg *)args->in_args);
- return fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
+ if (argbuf->is_flat)
+ return offset;
+ return round_up(offset, PAGE_SIZE);
}
static void virtio_fs_argbuf_copy_to_out_arg(struct virtio_fs_argbuf *argbuf,
unsigned int offset, void *dst,
unsigned int len)
{
- memcpy(dst, argbuf->buf + offset, len);
+ struct iov_iter iter;
+ unsigned int copied;
+
+ if (argbuf->is_flat) {
+ memcpy(dst, argbuf->f.buf + offset, len);
+ return;
+ }
+
+ iov_iter_bvec(&iter, ITER_SOURCE, argbuf->s.bvec,
+ argbuf->s.nr, argbuf->s.size);
+ iov_iter_advance(&iter, offset);
+
+ copied = _copy_from_iter(dst, len, &iter);
+ WARN_ON_ONCE(copied != len);
}
/*
@@ -1154,7 +1289,7 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg,
len = fuse_len_args(numargs - argpages, args);
if (len)
total_sgs += virtio_fs_argbuf_setup_sg(req->argbuf, *len_used,
- len, &sg[total_sgs]);
+ &len, &sg[total_sgs]);
if (argpages)
total_sgs += sg_init_fuse_pages(&sg[total_sgs],
@@ -1199,7 +1334,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
}
/* Use a bounce buffer since stack args cannot be mapped */
- req->argbuf = virtio_fs_argbuf_new(args, GFP_ATOMIC);
+ req->argbuf = virtio_fs_argbuf_new(args, GFP_ATOMIC, true);
if (!req->argbuf) {
ret = -ENOMEM;
goto out;
--
2.29.2
