From: Eelco Chaudron <echaudro@xxxxxxxxxx>
This change adds support for tail growing and shrinking for XDP multi-buff.
Signed-off-by: Eelco Chaudron <echaudro@xxxxxxxxxx>
Signed-off-by: Lorenzo Bianconi <lorenzo@xxxxxxxxxx>
---
include/net/xdp.h | 5 ++++
net/core/filter.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 68 insertions(+)
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 8be1b5e5a08a..19cd6642e087 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -157,6 +157,11 @@ static inline void xdp_set_frag_size(skb_frag_t *frag, u32 size)
frag->bv_len = size;
}
+static inline unsigned int xdp_get_frag_tailroom(const skb_frag_t *frag)
+{
+ return PAGE_SIZE - xdp_get_frag_size(frag) - xdp_get_frag_offset(frag);
+}
+
struct xdp_frame {
void *data;
u16 len;
diff --git a/net/core/filter.c b/net/core/filter.c
index 10dac9dd5086..18b2c9bacba1 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -3855,11 +3855,74 @@ static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
.arg2_type = ARG_ANYTHING,
};
+static int bpf_xdp_mb_adjust_tail(struct xdp_buff *xdp, int offset)
+{
+ struct xdp_shared_info *xdp_sinfo = xdp_get_shared_info_from_buff(xdp);
+
+ if (unlikely(xdp_sinfo->nr_frags == 0))
+ return -EINVAL;
+
+ if (offset >= 0) {
+ skb_frag_t *frag = &xdp_sinfo->frags[xdp_sinfo->nr_frags - 1];
+ int size;
+
+ if (unlikely(offset > xdp_get_frag_tailroom(frag)))
+ return -EINVAL;
+
+ size = xdp_get_frag_size(frag);
+ memset(xdp_get_frag_address(frag) + size, 0, offset);
+ xdp_set_frag_size(frag, size + offset);
+ xdp_sinfo->data_length += offset;
+ } else {
+ int i, frags_to_free = 0;
+
+ offset = abs(offset);
+
+ if (unlikely(offset > ((int)(xdp->data_end - xdp->data) +
+ xdp_sinfo->data_length -
+ ETH_HLEN)))
+ return -EINVAL;
+
+ for (i = xdp_sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) {
+ skb_frag_t *frag = &xdp_sinfo->frags[i];
+ int size = xdp_get_frag_size(frag);
+ int shrink = min_t(int, offset, size);
+
+ offset -= shrink;
+ if (likely(size - shrink > 0)) {
+ /* When updating the final fragment we have
+ * to adjust the data_length in line.
+ */
+ xdp_sinfo->data_length -= shrink;
+ xdp_set_frag_size(frag, size - shrink);
+ break;
+ }
+
+ /* When we free the fragments,
+ * xdp_return_frags_from_buff() will take care
+ * of updating the xdp share info data_length.
+ */
+ frags_to_free++;
+ }
+
+ if (unlikely(frags_to_free))
+ xdp_return_num_frags_from_buff(xdp, frags_to_free);
+
+ if (unlikely(offset > 0))
+ xdp->data_end -= offset;
+ }
+
+ return 0;
+}
+
BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset)
{
void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */
void *data_end = xdp->data_end + offset;
+ if (unlikely(xdp->mb))
+ return bpf_xdp_mb_adjust_tail(xdp, offset);
+
/* Notice that xdp_data_hard_end have reserved some tailroom */
if (unlikely(data_end > data_hard_end))
return -EINVAL;
--
2.30.2
