Split copy_from_iter_flushcache() by type.
Signed-off-by: David Howells <dhowells@xxxxxxxxxx>
---
lib/iov_iter.c | 42 +++++++++++++++++++++++++-----------------
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index c57c2171f730..6b4739d7dd9a 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1000,7 +1000,7 @@ static size_t kvec_copy_from_iter_nocache(void *addr, size_t bytes, struct iov_i
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
/**
- * _copy_from_iter_flushcache - write destination through cpu cache
+ * copy_from_iter_flushcache - write destination through cpu cache
* @addr: destination kernel address
* @bytes: total transfer length
* @iter: source iterator
@@ -1013,22 +1013,30 @@ static size_t kvec_copy_from_iter_nocache(void *addr, size_t bytes, struct iov_i
* bypass the cache for the ITER_IOVEC case, and on some archs may use
* instructions that strand dirty-data in the cache.
*/
-static size_t xxx_copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
+static size_t iovec_copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
{
char *to = addr;
- if (unlikely(iov_iter_is_pipe(i))) {
- WARN_ON(1);
- return 0;
- }
- iterate_and_advance(i, bytes, v,
+ iterate_and_advance_iovec(i, bytes, v,
__copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
- v.iov_base, v.iov_len),
+ v.iov_base, v.iov_len));
+ return bytes;
+}
+
+static size_t bvec_copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ iterate_and_advance_bvec(i, bytes, v,
memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
- v.bv_offset, v.bv_len),
- memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
- v.iov_len)
- )
+ v.bv_offset, v.bv_len));
+ return bytes;
+}
+static size_t kvec_copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
+{
+ char *to = addr;
+ iterate_and_advance_kvec(i, bytes, v,
+ memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
+ v.iov_len));
return bytes;
}
#endif
@@ -2020,7 +2028,7 @@ static const struct iov_iter_ops iovec_iter_ops = {
.copy_from_iter_nocache = iovec_copy_from_iter_nocache,
.copy_from_iter_full_nocache = xxx_copy_from_iter_full_nocache,
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
- .copy_from_iter_flushcache = xxx_copy_from_iter_flushcache,
+ .copy_from_iter_flushcache = iovec_copy_from_iter_flushcache,
#endif
#ifdef CONFIG_ARCH_HAS_COPY_MC
.copy_mc_to_iter = iovec_copy_mc_to_iter,
@@ -2054,7 +2062,7 @@ static const struct iov_iter_ops kvec_iter_ops = {
.copy_from_iter_nocache = kvec_copy_from_iter_nocache,
.copy_from_iter_full_nocache = xxx_copy_from_iter_full_nocache,
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
- .copy_from_iter_flushcache = xxx_copy_from_iter_flushcache,
+ .copy_from_iter_flushcache = kvec_copy_from_iter_flushcache,
#endif
#ifdef CONFIG_ARCH_HAS_COPY_MC
.copy_mc_to_iter = kvec_copy_mc_to_iter,
@@ -2088,7 +2096,7 @@ static const struct iov_iter_ops bvec_iter_ops = {
.copy_from_iter_nocache = bvec_copy_from_iter_nocache,
.copy_from_iter_full_nocache = xxx_copy_from_iter_full_nocache,
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
- .copy_from_iter_flushcache = xxx_copy_from_iter_flushcache,
+ .copy_from_iter_flushcache = bvec_copy_from_iter_flushcache,
#endif
#ifdef CONFIG_ARCH_HAS_COPY_MC
.copy_mc_to_iter = bvec_copy_mc_to_iter,
@@ -2122,7 +2130,7 @@ static const struct iov_iter_ops pipe_iter_ops = {
.copy_from_iter_nocache = no_copy_from_iter,
.copy_from_iter_full_nocache = xxx_copy_from_iter_full_nocache,
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
- .copy_from_iter_flushcache = xxx_copy_from_iter_flushcache,
+ .copy_from_iter_flushcache = no_copy_from_iter,
#endif
#ifdef CONFIG_ARCH_HAS_COPY_MC
.copy_mc_to_iter = pipe_copy_mc_to_iter,
@@ -2156,7 +2164,7 @@ static const struct iov_iter_ops discard_iter_ops = {
.copy_from_iter_nocache = no_copy_from_iter,
.copy_from_iter_full_nocache = xxx_copy_from_iter_full_nocache,
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
- .copy_from_iter_flushcache = xxx_copy_from_iter_flushcache,
+ .copy_from_iter_flushcache = no_copy_from_iter,
#endif
#ifdef CONFIG_ARCH_HAS_COPY_MC
.copy_mc_to_iter = discard_copy_to_iter,
