Renumber the ITER_* constants in uio.h to be contiguous to make comparing
them more efficient in a switch-statement.
Signed-off-by: David Howells <dhowelsl@xxxxxxxxxx>
---
include/linux/uio.h | 8 +++---
lib/iov_iter.c | 69 +++++++++++++++++++++++++++++++++++++++++----------
2 files changed, 60 insertions(+), 17 deletions(-)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 48e7fa36f923..d5f8755bf778 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -22,10 +22,10 @@ struct kvec {
};
enum iter_type {
- ITER_IOVEC = 0,
- ITER_KVEC = 2,
- ITER_BVEC = 4,
- ITER_PIPE = 8,
+ ITER_IOVEC,
+ ITER_KVEC,
+ ITER_BVEC,
+ ITER_PIPE,
};
struct iov_iter {
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index bd828591afb0..f30ecd263d6e 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -75,18 +75,28 @@
#define iterate_all_kinds(i, n, v, I, B, K) { \
if (likely(n)) { \
size_t skip = i->iov_offset; \
- if (unlikely(i->iter_type & ITER_BVEC)) { \
+ switch (iov_iter_type(i)) { \
+ case ITER_BVEC: { \
struct bio_vec v; \
struct bvec_iter __bi; \
- iterate_bvec(i, n, v, __bi, skip, (B)) \
- } else if (unlikely(i->iter_type & ITER_KVEC)) { \
+ iterate_bvec(i, n, v, __bi, skip, (B)); \
+ break; \
+ } \
+ case ITER_KVEC: { \
const struct kvec *kvec; \
struct kvec v; \
- iterate_kvec(i, n, v, kvec, skip, (K)) \
- } else { \
+ iterate_kvec(i, n, v, kvec, skip, (K)); \
+ break; \
+ } \
+ case ITER_PIPE: { \
+ break; \
+ } \
+ case ITER_IOVEC: { \
const struct iovec *iov; \
struct iovec v; \
- iterate_iovec(i, n, v, iov, skip, (I)) \
+ iterate_iovec(i, n, v, iov, skip, (I)); \
+ break; \
+ } \
} \
} \
}
@@ -96,7 +106,8 @@
n = i->count; \
if (i->count) { \
size_t skip = i->iov_offset; \
- if (unlikely(i->iter_type & ITER_BVEC)) { \
+ switch (iov_iter_type(i)) { \
+ case ITER_BVEC: { \
const struct bio_vec *bvec = i->bvec; \
struct bio_vec v; \
struct bvec_iter __bi; \
@@ -104,7 +115,9 @@
i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
i->nr_segs -= i->bvec - bvec; \
skip = __bi.bi_bvec_done; \
- } else if (unlikely(i->iter_type & ITER_KVEC)) { \
+ break; \
+ } \
+ case ITER_KVEC: { \
const struct kvec *kvec; \
struct kvec v; \
iterate_kvec(i, n, v, kvec, skip, (K)) \
@@ -114,7 +127,9 @@
} \
i->nr_segs -= kvec - i->kvec; \
i->kvec = kvec; \
- } else { \
+ break; \
+ } \
+ case ITER_IOVEC: { \
const struct iovec *iov; \
struct iovec v; \
iterate_iovec(i, n, v, iov, skip, (I)) \
@@ -124,6 +139,11 @@
} \
i->nr_segs -= iov - i->iov; \
i->iov = iov; \
+ break; \
+ } \
+ case ITER_PIPE: { \
+ break; \
+ } \
} \
i->count -= n; \
i->iov_offset = skip; \
@@ -873,6 +893,7 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
case ITER_IOVEC:
return copy_page_from_iter_iovec(page, offset, bytes, i);
}
+
WARN_ON(1);
return 0;
}
@@ -988,11 +1009,17 @@ static void pipe_advance(struct iov_iter *i, size_t size)
void iov_iter_advance(struct iov_iter *i, size_t size)
{
- if (unlikely(iov_iter_is_pipe(i))) {
+ switch (iov_iter_type(i)) {
+ case ITER_PIPE:
pipe_advance(i, size);
return;
+ case ITER_IOVEC:
+ case ITER_KVEC:
+ case ITER_BVEC:
+ iterate_and_advance(i, size, v, 0, 0, 0);
+ return;
}
- iterate_and_advance(i, size, v, 0, 0, 0)
+ BUG();
}
EXPORT_SYMBOL(iov_iter_advance);
@@ -1223,8 +1250,16 @@ ssize_t iov_iter_get_pages(struct iov_iter *i,
if (maxsize > i->count)
maxsize = i->count;
- if (unlikely(iov_iter_is_pipe(i)))
+ switch (iov_iter_type(i)) {
+ case ITER_PIPE:
return pipe_get_pages(i, pages, maxsize, maxpages, start);
+ case ITER_KVEC:
+ return -EFAULT;
+ case ITER_IOVEC:
+ case ITER_BVEC:
+ break;
+ }
+
iterate_all_kinds(i, maxsize, v, ({
unsigned long addr = (unsigned long)v.iov_base;
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
@@ -1300,8 +1335,16 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
if (maxsize > i->count)
maxsize = i->count;
- if (unlikely(iov_iter_is_pipe(i)))
+ switch (iov_iter_type(i)) {
+ case ITER_PIPE:
return pipe_get_pages_alloc(i, pages, maxsize, start);
+ case ITER_KVEC:
+ return -EFAULT;
+ case ITER_IOVEC:
+ case ITER_BVEC:
+ break;
+ }
+
iterate_all_kinds(i, maxsize, v, ({
unsigned long addr = (unsigned long)v.iov_base;
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
