Add an iterator type that can iterate over a socket buffer.
[!] Note this is not yet completely implemented and won't compile.
Signed-off-by: David Howells <dhowells@xxxxxxxxxx>
---
include/linux/uio.h | 10 ++++
lib/iov_iter.c | 121 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 131 insertions(+)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 0e50f4af6877..87d6ba660489 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -13,6 +13,7 @@
struct page;
struct folio_queue;
struct scatterlist;
+struct sk_buff;
typedef unsigned int __bitwise iov_iter_extraction_t;
@@ -32,6 +33,7 @@ enum iter_type {
ITER_DISCARD,
ITER_ITERLIST,
ITER_SCATTERLIST,
+ ITER_SKBUFF,
};
#define ITER_SOURCE 1 // == WRITE
@@ -77,6 +79,7 @@ struct iov_iter {
void __user *ubuf;
struct iov_iterlist *iterlist;
struct scatterlist *sglist;
+ const struct sk_buff *skb;
};
size_t count;
};
@@ -171,6 +174,11 @@ static inline bool iov_iter_is_scatterlist(const struct iov_iter *i)
return iov_iter_type(i) == ITER_SCATTERLIST;
}
+static inline bool iov_iter_is_skbuff(const struct iov_iter *i)
+{
+ return iov_iter_type(i) == ITER_SKBUFF;
+}
+
static inline unsigned char iov_iter_rw(const struct iov_iter *i)
{
return i->data_source ? WRITE : READ;
@@ -329,6 +337,8 @@ void iov_iter_iterlist(struct iov_iter *i, unsigned int direction,
size_t count);
void iov_iter_scatterlist(struct iov_iter *i, unsigned int direction,
struct scatterlist *sglist, size_t count);
+void iov_iter_skbuff(struct iov_iter *i, unsigned int direction,
+ const struct sk_buff *skb, size_t count);
ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,
size_t maxsize, unsigned maxpages, size_t *start);
ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages,
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index ed9859af3c5d..01215316d272 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -12,6 +12,7 @@
#include <linux/scatterlist.h>
#include <linux/instrumented.h>
#include <linux/iov_iter.h>
+#include <linux/skbuff.h>
static __always_inline
size_t copy_to_user_iter(void __user *iter_to, size_t progress,
@@ -918,6 +919,29 @@ void iov_iter_scatterlist(struct iov_iter *iter, unsigned int direction,
}
EXPORT_SYMBOL(iov_iter_scatterlist);
+/**
+ * iov_iter_skbuff - Initialise an I/O iterator for a socket buffer
+ * @iter: The iterator to initialise.
+ * @direction: The direction of the transfer.
+ * @skb: The socket buffer
+ * @count: The size of the I/O buffer in bytes.
+ *
+ * Set up an I/O iterator that walks over a socket buffer.
+ */
+void iov_iter_skbuff(struct iov_iter *i, unsigned int direction,
+ const struct sk_buff *skb, size_t count)
+{
+ WARN_ON(direction & ~(READ | WRITE));
+ *iter = (struct iov_iter){
+ .iter_type = ITER_SKBUFF,
+ .data_source = direction,
+ .skb = skb,
+ .iov_offset = 0,
+ .count = count,
+ };
+}
+EXPORT_SYMBOL(iov_iter_skbuff);
+
static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
unsigned len_mask)
{
@@ -2314,6 +2338,10 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i,
return iov_iter_extract_scatterlist_pages(i, pages, maxsize,
maxpages, extraction_flags,
offset0);
+ if (iov_iter_is_skbuff(i))
+ return iov_iter_extract_skbuff_pages(i, pages, maxsize,
+ maxpages, extraction_flags,
+ offset0);
return -EFAULT;
}
EXPORT_SYMBOL_GPL(iov_iter_extract_pages);
@@ -2449,6 +2477,97 @@ static size_t iterate_scatterlist(struct iov_iter *iter, size_t len, void *priv,
return progress;
}
+struct skbuff_iter_ctx {
+ iov_step_f step;
+ size_t progress;
+ void *priv;
+ void *priv2;
+};
+
+static bool iterate_skbuff_frag(const struct sk_buff *skb, struct skbuff_iter_ctx *ctx,
+ int offset, int len, int recursion_level)
+{
+ struct sk_buff *frag_iter;
+ size_t skip = offset, part, remain, consumed;
+
+ if (unlikely(recursion_level >= 24))
+ return false;
+
+ part = skb_headlen(skb);
+ if (skip < part) {
+ part = umin(part - skip, len);
+ remain = ctx->step(skb->data + skip, ctx->progress, part,
+ ctx->priv, ctx->priv2);
+ consumed = part - remain;
+ ctx->progress += consumed;
+ len -= consumed;
+ if (remain > 0 || len <= 0)
+ return false;
+ skip = 0;
+ } else {
+ skip -= part;
+ }
+
+ for (int i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+ const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+ size_t fsize = skb_frag_size(frag);
+
+ if (skip >= fsize) {
+ skip -= fsize;
+ continue;
+ }
+
+ part = umin(fsize - skip, len);
+ remain = ctx->step(skb_frag_address(frag) + skip,
+ ctx->progress, part, ctx->priv, ctx->priv2);
+ consumed = part - remain;
+ ctx->progress += consumed;
+ len -= consumed;
+ if (remain > 0 || len <= 0)
+ return false;
+ skip = 0;
+ }
+
+ skb_walk_frags(skb, frag_iter) {
+ size_t fsize = frag_iter->len;
+
+ if (skip >= fsize) {
+ skip -= fsize;
+ continue;
+ }
+
+ part = umin(fsize - skip, len);
+ if (!iterate_skbuff_frag(frag_iter, ctx, skb_headlen(skb) + skip,
+ part, recursion_level + 1))
+ return false;
+ len -= part;
+ if (len <= 0)
+ return false;
+ skip = 0;
+ }
+ return true;
+}
+
+/*
+ * Handle iteration over ITER_SKBUFF. Modelled on __skb_to_sgvec().
+ */
+static size_t iterate_skbuff(struct iov_iter *iter, size_t len, void *priv, void *priv2,
+ iov_step_f step)
+{
+ struct skbuff_iter_ctx ctx = {
+ .step = step,
+ .progress = 0,
+ .priv = priv,
+ .priv2 = priv2,
+ };
+
+ iterate_skbuff_frag(iter->skb, &ctx, iter->iov_offset, len, 0);
+
+ iter->iov_offset += ctx.progress;
+ iter->count -= ctx.progress;
+ return ctx.progress;
+}
+
/*
* Out of line iteration for iterator types that don't need such fast handling.
*/
@@ -2463,6 +2582,8 @@ size_t __iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv,
return iterate_iterlist(iter, len, priv, priv2, ustep, step);
if (iov_iter_is_scatterlist(iter))
return iterate_scatterlist(iter, len, priv, priv2, step);
+ if (iov_iter_is_skbuff(iter))
+ return iterate_skbuff(iter, len, priv, priv2, step);
WARN_ON(1);
return 0;
}
