On Wed, 2022-06-22 at 05:15 +0100, Al Viro wrote:
> All call sites of get_pages_array() are essenitally identical now.
> Replace with common helper...
>
> Returns number of slots available in resulting array or 0 on OOM;
> it's up to the caller to make sure it doesn't ask to zero-entry
> array (i.e. neither maxpages nor size are allowed to be zero).
>
> Signed-off-by: Al Viro <viro@xxxxxxxxxxxxxxxxxx>
> ---
> lib/iov_iter.c | 77 +++++++++++++++++++++-----------------------------
> 1 file changed, 32 insertions(+), 45 deletions(-)
>
> diff --git a/lib/iov_iter.c b/lib/iov_iter.c
> index 9280f865fd6a..1c744f0c0b2c 100644
> --- a/lib/iov_iter.c
> +++ b/lib/iov_iter.c
> @@ -1187,9 +1187,20 @@ unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
> }
> EXPORT_SYMBOL(iov_iter_gap_alignment);
>
> -static struct page **get_pages_array(size_t n)
> +static int want_pages_array(struct page ***res, size_t size,
> + size_t start, unsigned int maxpages)
> {
> - return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
> + unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE);
> +
> + if (count > maxpages)
> + count = maxpages;
> + WARN_ON(!count); // caller should've prevented that
> + if (!*res) {
> + *res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);
> + if (!*res)
> + return 0;
> + }
> + return count;
> }
>
> static ssize_t pipe_get_pages(struct iov_iter *i,
> @@ -1197,27 +1208,20 @@ static ssize_t pipe_get_pages(struct iov_iter *i,
> size_t *start)
> {
> struct pipe_inode_info *pipe = i->pipe;
> - unsigned int npages, off;
> + unsigned int npages, off, count;
> struct page **p;
> ssize_t left;
> - int count;
>
> if (!sanity(i))
> return -EFAULT;
>
> *start = off = pipe_npages(i, &npages);
> - count = DIV_ROUND_UP(maxsize + off, PAGE_SIZE);
> - if (count > npages)
> - count = npages;
> - if (count > maxpages)
> - count = maxpages;
> + if (!npages)
> + return -EFAULT;
> + count = want_pages_array(pages, maxsize, off, min(npages, maxpages));
> + if (!count)
> + return -ENOMEM;
> p = *pages;
> - if (!p) {
> - *pages = p = get_pages_array(count);
> - if (!p)
> - return -ENOMEM;
> - }
> -
> left = maxsize;
> npages = 0;
> if (off) {
> @@ -1280,9 +1284,8 @@ static ssize_t iter_xarray_get_pages(struct iov_iter *i,
> struct page ***pages, size_t maxsize,
> unsigned maxpages, size_t *_start_offset)
> {
> - unsigned nr, offset;
> - pgoff_t index, count;
> - size_t size = maxsize;
> + unsigned nr, offset, count;
> + pgoff_t index;
> loff_t pos;
>
> pos = i->xarray_start + i->iov_offset;
> @@ -1290,16 +1293,9 @@ static ssize_t iter_xarray_get_pages(struct iov_iter *i,
> offset = pos & ~PAGE_MASK;
> *_start_offset = offset;
>
> - count = DIV_ROUND_UP(size + offset, PAGE_SIZE);
> - if (count > maxpages)
> - count = maxpages;
> -
> - if (!*pages) {
> - *pages = get_pages_array(count);
> - if (!*pages)
> - return -ENOMEM;
> - }
> -
> + count = want_pages_array(pages, maxsize, offset, maxpages);
> + if (!count)
> + return -ENOMEM;
> nr = iter_xarray_populate_pages(*pages, i->xarray, index, count);
> if (nr == 0)
> return 0;
> @@ -1348,7 +1344,7 @@ static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
> struct page ***pages, size_t maxsize,
> unsigned int maxpages, size_t *start)
> {
> - int n, res;
> + unsigned int n;
>
> if (maxsize > i->count)
> maxsize = i->count;
> @@ -1360,6 +1356,7 @@ static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
> if (likely(user_backed_iter(i))) {
> unsigned int gup_flags = 0;
> unsigned long addr;
> + int res;
>
> if (iov_iter_rw(i) != WRITE)
> gup_flags |= FOLL_WRITE;
> @@ -1369,14 +1366,9 @@ static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
> addr = first_iovec_segment(i, &maxsize);
> *start = addr % PAGE_SIZE;
> addr &= PAGE_MASK;
> - n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
> - if (n > maxpages)
> - n = maxpages;
> - if (!*pages) {
> - *pages = get_pages_array(n);
> - if (!*pages)
> - return -ENOMEM;
> - }
> + n = want_pages_array(pages, maxsize, *start, maxpages);
> + if (!n)
> + return -ENOMEM;
> res = get_user_pages_fast(addr, n, gup_flags, *pages);
> if (unlikely(res <= 0))
> return res;
> @@ -1387,15 +1379,10 @@ static ssize_t __iov_iter_get_pages_alloc(struct iov_iter *i,
> struct page *page;
>
> page = first_bvec_segment(i, &maxsize, start);
> - n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
> - if (n > maxpages)
> - n = maxpages;
> + n = want_pages_array(pages, maxsize, *start, maxpages);
> + if (!n)
> + return -ENOMEM;
> p = *pages;
> - if (!p) {
> - *pages = p = get_pages_array(n);
> - if (!p)
> - return -ENOMEM;
> - }
> for (int k = 0; k < n; k++)
> get_page(*p++ = page++);
> return min_t(size_t, maxsize, n * PAGE_SIZE - *start);
Reviewed-by: Jeff Layton <jlayton@xxxxxxxxxx>
