This adds a new helper to dequeue a packet from a PIFO map,
bpf_packet_dequeue(). The helper returns a refcounted pointer to the packet
dequeued from the map; the reference must be released either by dropping
the packet (using bpf_packet_drop()), or by returning it to the caller.
Signed-off-by: Toke Høiland-Jørgensen <toke@xxxxxxxxxx>
---
include/uapi/linux/bpf.h | 19 +++++++++++++++
kernel/bpf/verifier.c | 13 +++++++---
net/core/filter.c | 43 +++++++++++++++++++++++++++++++++-
tools/include/uapi/linux/bpf.h | 19 +++++++++++++++
4 files changed, 90 insertions(+), 4 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 974fb5882305..d44382644391 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -5341,6 +5341,23 @@ union bpf_attr {
* **-EACCES** if the SYN cookie is not valid.
*
* **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin.
+ *
+ * long bpf_packet_dequeue(void *ctx, struct bpf_map *map, u64 flags, u64 *rank)
+ * Description
+ * Dequeue the packet at the head of the PIFO in *map* and return a pointer
+ * to the packet (or NULL if the PIFO is empty).
+ * Return
+ * On success, a pointer to the packet, or NULL if the PIFO is empty. The
+ * packet pointer must be freed using *bpf_packet_drop()* or returning
+ * the packet pointer. The *rank* pointer will be set to the rank of
+ * the dequeued packet on success, or a negative error code on error.
+ *
+ * long bpf_packet_drop(void *ctx, void *pkt)
+ * Description
+ * Drop *pkt*, which must be a reference previously returned by
+ * *bpf_packet_dequeue()* (and checked to not be NULL).
+ * Return
+ * This always succeeds and returns zero.
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -5551,6 +5568,8 @@ union bpf_attr {
FN(tcp_raw_gen_syncookie_ipv6), \
FN(tcp_raw_check_syncookie_ipv4), \
FN(tcp_raw_check_syncookie_ipv6), \
+ FN(packet_dequeue), \
+ FN(packet_drop), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e3662460a095..68f98d76bc78 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -483,7 +483,8 @@ static bool may_be_acquire_function(enum bpf_func_id func_id)
func_id == BPF_FUNC_sk_lookup_udp ||
func_id == BPF_FUNC_skc_lookup_tcp ||
func_id == BPF_FUNC_map_lookup_elem ||
- func_id == BPF_FUNC_ringbuf_reserve;
+ func_id == BPF_FUNC_ringbuf_reserve ||
+ func_id == BPF_FUNC_packet_dequeue;
}
static bool is_acquire_function(enum bpf_func_id func_id,
@@ -495,7 +496,8 @@ static bool is_acquire_function(enum bpf_func_id func_id,
func_id == BPF_FUNC_sk_lookup_udp ||
func_id == BPF_FUNC_skc_lookup_tcp ||
func_id == BPF_FUNC_ringbuf_reserve ||
- func_id == BPF_FUNC_kptr_xchg)
+ func_id == BPF_FUNC_kptr_xchg ||
+ func_id == BPF_FUNC_packet_dequeue)
return true;
if (func_id == BPF_FUNC_map_lookup_elem &&
@@ -6276,7 +6278,8 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
goto error;
break;
case BPF_MAP_TYPE_PIFO_XDP:
- if (func_id != BPF_FUNC_redirect_map)
+ if (func_id != BPF_FUNC_redirect_map &&
+ func_id != BPF_FUNC_packet_dequeue)
goto error;
break;
default:
@@ -6385,6 +6388,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
goto error;
break;
+ case BPF_FUNC_packet_dequeue:
+ if (map->map_type != BPF_MAP_TYPE_PIFO_XDP)
+ goto error;
+ break;
default:
break;
}
diff --git a/net/core/filter.c b/net/core/filter.c
index 30bd3a6aedab..893b75515859 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4430,6 +4430,40 @@ static const struct bpf_func_proto bpf_xdp_redirect_map_proto = {
.arg3_type = ARG_ANYTHING,
};
+BTF_ID_LIST_SINGLE(xdp_md_btf_ids, struct, xdp_md)
+
+BPF_CALL_4(bpf_packet_dequeue, struct dequeue_data *, ctx, struct bpf_map *, map,
+ u64, flags, u64 *, rank)
+{
+ return (unsigned long)pifo_map_dequeue(map, flags, rank);
+}
+
+static const struct bpf_func_proto bpf_packet_dequeue_proto = {
+ .func = bpf_packet_dequeue,
+ .gpl_only = false,
+ .ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
+ .ret_btf_id = xdp_md_btf_ids,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_CONST_MAP_PTR,
+ .arg3_type = ARG_ANYTHING,
+ .arg4_type = ARG_PTR_TO_LONG,
+};
+
+BPF_CALL_2(bpf_packet_drop, struct dequeue_data *, ctx, struct xdp_frame *, pkt)
+{
+ xdp_return_frame(pkt);
+ return 0;
+}
+
+static const struct bpf_func_proto bpf_packet_drop_proto = {
+ .func = bpf_packet_drop,
+ .gpl_only = false,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_CTX,
+ .arg2_type = ARG_PTR_TO_BTF_ID | OBJ_RELEASE,
+ .arg2_btf_id = xdp_md_btf_ids,
+};
+
static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
unsigned long off, unsigned long len)
{
@@ -8065,7 +8099,14 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
static const struct bpf_func_proto *
dequeue_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
- return bpf_base_func_proto(func_id);
+ switch (func_id) {
+ case BPF_FUNC_packet_dequeue:
+ return &bpf_packet_dequeue_proto;
+ case BPF_FUNC_packet_drop:
+ return &bpf_packet_drop_proto;
+ default:
+ return bpf_base_func_proto(func_id);
+ }
}
const struct bpf_func_proto bpf_sock_map_update_proto __weak;
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 4dd8a563f85d..1dab68a89e18 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -5341,6 +5341,23 @@ union bpf_attr {
* **-EACCES** if the SYN cookie is not valid.
*
* **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin.
+ *
+ * long bpf_packet_dequeue(void *ctx, struct bpf_map *map, u64 flags, u64 *rank)
+ * Description
+ * Dequeue the packet at the head of the PIFO in *map* and return a pointer
+ * to the packet (or NULL if the PIFO is empty).
+ * Return
+ * On success, a pointer to the packet, or NULL if the PIFO is empty. The
+ * packet pointer must be freed using *bpf_packet_drop()* or returning
+ * the packet pointer. The *rank* pointer will be set to the rank of
+ * the dequeued packet on success, or a negative error code on error.
+ *
+ * long bpf_packet_drop(void *ctx, void *pkt)
+ * Description
+ * Drop *pkt*, which must be a reference previously returned by
+ * *bpf_packet_dequeue()* (and checked to not be NULL).
+ * Return
+ * This always succeeds and returns zero.
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -5551,6 +5568,8 @@ union bpf_attr {
FN(tcp_raw_gen_syncookie_ipv6), \
FN(tcp_raw_check_syncookie_ipv4), \
FN(tcp_raw_check_syncookie_ipv6), \
+ FN(packet_dequeue), \
+ FN(packet_drop), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
--
2.37.0
