From: Jens Axboe <axboe@xxxxxxxxx>
Commit e270bfd22a2a10d1cfbaddf23e79b6d0b405d21e upstream.
This avoids needing to care about HIGHMEM, and it makes the buffer
indexing easier as both ring provided buffer methods are now virtually
mapped in a contigious fashion.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx>
---
io_uring/kbuf.c | 39 +++++++++++++++------------------------
1 file changed, 15 insertions(+), 24 deletions(-)
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -7,6 +7,7 @@
#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/poll.h>
+#include <linux/vmalloc.h>
#include <linux/io_uring.h>
#include <uapi/linux/io_uring.h>
@@ -153,15 +154,7 @@ static void __user *io_ring_buffer_selec
return NULL;
head &= bl->mask;
- /* mmaped buffers are always contig */
- if (bl->is_mmap || head < IO_BUFFER_LIST_BUF_PER_PAGE) {
- buf = &br->bufs[head];
- } else {
- int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE - 1);
- int index = head / IO_BUFFER_LIST_BUF_PER_PAGE;
- buf = page_address(bl->buf_pages[index]);
- buf += off;
- }
+ buf = &br->bufs[head];
if (*len == 0 || *len > buf->len)
*len = buf->len;
req->flags |= REQ_F_BUFFER_RING;
@@ -249,6 +242,7 @@ static int __io_remove_buffers(struct io
for (j = 0; j < bl->buf_nr_pages; j++)
unpin_user_page(bl->buf_pages[j]);
kvfree(bl->buf_pages);
+ vunmap(bl->buf_ring);
bl->buf_pages = NULL;
bl->buf_nr_pages = 0;
}
@@ -501,9 +495,9 @@ err:
static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg,
struct io_buffer_list *bl)
{
- struct io_uring_buf_ring *br;
+ struct io_uring_buf_ring *br = NULL;
+ int nr_pages, ret, i;
struct page **pages;
- int i, nr_pages;
pages = io_pin_pages(reg->ring_addr,
flex_array_size(br, bufs, reg->ring_entries),
@@ -511,18 +505,12 @@ static int io_pin_pbuf_ring(struct io_ur
if (IS_ERR(pages))
return PTR_ERR(pages);
- /*
- * Apparently some 32-bit boxes (ARM) will return highmem pages,
- * which then need to be mapped. We could support that, but it'd
- * complicate the code and slowdown the common cases quite a bit.
- * So just error out, returning -EINVAL just like we did on kernels
- * that didn't support mapped buffer rings.
- */
- for (i = 0; i < nr_pages; i++)
- if (PageHighMem(pages[i]))
- goto error_unpin;
+ br = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
+ if (!br) {
+ ret = -ENOMEM;
+ goto error_unpin;
+ }
- br = page_address(pages[0]);
#ifdef SHM_COLOUR
/*
* On platforms that have specific aliasing requirements, SHM_COLOUR
@@ -533,8 +521,10 @@ static int io_pin_pbuf_ring(struct io_ur
* should use IOU_PBUF_RING_MMAP instead, and liburing will handle
* this transparently.
*/
- if ((reg->ring_addr | (unsigned long) br) & (SHM_COLOUR - 1))
+ if ((reg->ring_addr | (unsigned long) br) & (SHM_COLOUR - 1)) {
+ ret = -EINVAL;
goto error_unpin;
+ }
#endif
bl->buf_pages = pages;
bl->buf_nr_pages = nr_pages;
@@ -546,7 +536,8 @@ error_unpin:
for (i = 0; i < nr_pages; i++)
unpin_user_page(pages[i]);
kvfree(pages);
- return -EINVAL;
+ vunmap(br);
+ return ret;
}
/*
Patches currently in stable-queue which might be from axboe@xxxxxxxxx are
queue-6.6/io_uring-fix-corner-case-forgetting-to-vunmap.patch
queue-6.6/io-wq-backoff-when-retrying-worker-creation.patch
queue-6.6/io_uring-kbuf-use-vm_insert_pages-for-mmap-ed-pbuf-ring.patch
queue-6.6/io_uring-use-vmap-for-ring-mapping.patch
queue-6.6/io_uring-get-rid-of-remap_pfn_range-for-mapping-rings-sqes.patch
queue-6.6/mm-add-nommu-variant-of-vm_insert_pages.patch
queue-6.6/zram-fix-null-pointer-in-comp_algorithm_show.patch
queue-6.6/io_uring-fix-error-pbuf-checking.patch
queue-6.6/io_uring-use-unpin_user_pages-where-appropriate.patch
queue-6.6/io_uring-kbuf-vmap-pinned-buffer-ring.patch
queue-6.6/io_uring-don-t-attempt-to-mmap-larger-than-what-the-user-asks-for.patch
queue-6.6/io_uring-unify-io_pin_pages.patch
queue-6.6/block-fix-kmem_cache-of-name-bio-108-already-exists.patch
queue-6.6/io_uring-add-ring-freeing-helper.patch
queue-6.6/io_uring-return-error-pointer-from-io_mem_alloc.patch
