Namely, move the following kfuncs:
- bpf_dynptr_is_null
- bpf_dynptr_is_rdonly
- bpf_dynptr_size
- bpf_dynptr_slice
Thus allowing verifier to inline these functions.
Signed-off-by: Eduard Zingerman <eddyz87@xxxxxxxxx>
---
include/linux/bpf.h | 36 +++++++++-
kernel/bpf/Makefile | 1 +
kernel/bpf/helpers.c | 130 +---------------------------------
kernel/bpf/inlinable_kfuncs.c | 112 +++++++++++++++++++++++++++++
4 files changed, 148 insertions(+), 131 deletions(-)
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 75f57f791cd3..7ca53e165ab0 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1319,11 +1319,43 @@ enum bpf_dynptr_type {
BPF_DYNPTR_TYPE_XDP,
};
+/* Since the upper 8 bits of dynptr->size is reserved, the
+ * maximum supported size is 2^24 - 1.
+ */
+#define DYNPTR_MAX_SIZE ((1UL << 24) - 1)
+#define DYNPTR_TYPE_SHIFT 28
+#define DYNPTR_SIZE_MASK 0xFFFFFF
+#define DYNPTR_RDONLY_BIT BIT(31)
+
int bpf_dynptr_check_size(u32 size);
-u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
+
+static inline u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr)
+{
+ return ptr->size & DYNPTR_SIZE_MASK;
+}
+
const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
-bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
+
+static inline bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr)
+{
+ return ptr->size & DYNPTR_RDONLY_BIT;
+}
+
+static inline enum bpf_dynptr_type bpf_dynptr_get_type(const struct bpf_dynptr_kern *ptr)
+{
+ return (ptr->size & ~(DYNPTR_RDONLY_BIT)) >> DYNPTR_TYPE_SHIFT;
+}
+
+static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
+{
+ u32 size = __bpf_dynptr_size(ptr);
+
+ if (len > size || offset > size - len)
+ return -E2BIG;
+
+ return 0;
+}
#ifdef CONFIG_BPF_JIT
int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 3d7ee81c8e2e..e806b2ea5d81 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -63,6 +63,7 @@ LLC ?= $(LLVM_PREFIX)llc$(LLVM_SUFFIX)
# -fpatchable-function-entry=16,16 is $(PADDING_CFLAGS)
CFLAGS_REMOVE_inlinable_kfuncs.bpf.bc.o += $(CC_FLAGS_FTRACE)
CFLAGS_REMOVE_inlinable_kfuncs.bpf.bc.o += $(PADDING_CFLAGS)
+CFLAGS_inlinable_kfuncs.bpf.bc.o += -D__FOR_BPF
$(obj)/inlinable_kfuncs.bpf.bc.o: $(src)/inlinable_kfuncs.c
$(Q)$(CLANG) $(c_flags) -emit-llvm -c $< -o $@
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 395221e53832..75dae5d3f05e 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1641,19 +1641,6 @@ static const struct bpf_func_proto bpf_kptr_xchg_proto = {
.arg2_btf_id = BPF_PTR_POISON,
};
-/* Since the upper 8 bits of dynptr->size is reserved, the
- * maximum supported size is 2^24 - 1.
- */
-#define DYNPTR_MAX_SIZE ((1UL << 24) - 1)
-#define DYNPTR_TYPE_SHIFT 28
-#define DYNPTR_SIZE_MASK 0xFFFFFF
-#define DYNPTR_RDONLY_BIT BIT(31)
-
-bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr)
-{
- return ptr->size & DYNPTR_RDONLY_BIT;
-}
-
void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
{
ptr->size |= DYNPTR_RDONLY_BIT;
@@ -1664,16 +1651,6 @@ static void bpf_dynptr_set_type(struct bpf_dynptr_kern *ptr, enum bpf_dynptr_typ
ptr->size |= type << DYNPTR_TYPE_SHIFT;
}
-static enum bpf_dynptr_type bpf_dynptr_get_type(const struct bpf_dynptr_kern *ptr)
-{
- return (ptr->size & ~(DYNPTR_RDONLY_BIT)) >> DYNPTR_TYPE_SHIFT;
-}
-
-u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr)
-{
- return ptr->size & DYNPTR_SIZE_MASK;
-}
-
static void bpf_dynptr_set_size(struct bpf_dynptr_kern *ptr, u32 new_size)
{
u32 metadata = ptr->size & ~DYNPTR_SIZE_MASK;
@@ -1700,16 +1677,6 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
memset(ptr, 0, sizeof(*ptr));
}
-static int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len)
-{
- u32 size = __bpf_dynptr_size(ptr);
-
- if (len > size || offset > size - len)
- return -E2BIG;
-
- return 0;
-}
-
BPF_CALL_4(bpf_dynptr_from_mem, void *, data, u32, size, u64, flags, struct bpf_dynptr_kern *, ptr)
{
int err;
@@ -2540,76 +2507,8 @@ __bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid)
return p;
}
-/**
- * bpf_dynptr_slice() - Obtain a read-only pointer to the dynptr data.
- * @p: The dynptr whose data slice to retrieve
- * @offset: Offset into the dynptr
- * @buffer__opt: User-provided buffer to copy contents into. May be NULL
- * @buffer__szk: Size (in bytes) of the buffer if present. This is the
- * length of the requested slice. This must be a constant.
- *
- * For non-skb and non-xdp type dynptrs, there is no difference between
- * bpf_dynptr_slice and bpf_dynptr_data.
- *
- * If buffer__opt is NULL, the call will fail if buffer_opt was needed.
- *
- * If the intention is to write to the data slice, please use
- * bpf_dynptr_slice_rdwr.
- *
- * The user must check that the returned pointer is not null before using it.
- *
- * Please note that in the case of skb and xdp dynptrs, bpf_dynptr_slice
- * does not change the underlying packet data pointers, so a call to
- * bpf_dynptr_slice will not invalidate any ctx->data/data_end pointers in
- * the bpf program.
- *
- * Return: NULL if the call failed (eg invalid dynptr), pointer to a read-only
- * data slice (can be either direct pointer to the data or a pointer to the user
- * provided buffer, with its contents containing the data, if unable to obtain
- * direct pointer)
- */
__bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr *p, u32 offset,
- void *buffer__opt, u32 buffer__szk)
-{
- const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
- enum bpf_dynptr_type type;
- u32 len = buffer__szk;
- int err;
-
- if (!ptr->data)
- return NULL;
-
- err = bpf_dynptr_check_off_len(ptr, offset, len);
- if (err)
- return NULL;
-
- type = bpf_dynptr_get_type(ptr);
-
- switch (type) {
- case BPF_DYNPTR_TYPE_LOCAL:
- case BPF_DYNPTR_TYPE_RINGBUF:
- return ptr->data + ptr->offset + offset;
- case BPF_DYNPTR_TYPE_SKB:
- if (buffer__opt)
- return skb_header_pointer(ptr->data, ptr->offset + offset, len, buffer__opt);
- else
- return skb_pointer_if_linear(ptr->data, ptr->offset + offset, len);
- case BPF_DYNPTR_TYPE_XDP:
- {
- void *xdp_ptr = bpf_xdp_pointer(ptr->data, ptr->offset + offset, len);
- if (!IS_ERR_OR_NULL(xdp_ptr))
- return xdp_ptr;
-
- if (!buffer__opt)
- return NULL;
- bpf_xdp_copy_buf(ptr->data, ptr->offset + offset, buffer__opt, len, false);
- return buffer__opt;
- }
- default:
- WARN_ONCE(true, "unknown dynptr type %d\n", type);
- return NULL;
- }
-}
+ void *buffer__opt, u32 buffer__szk);
/**
* bpf_dynptr_slice_rdwr() - Obtain a writable pointer to the dynptr data.
@@ -2705,33 +2604,6 @@ __bpf_kfunc int bpf_dynptr_adjust(const struct bpf_dynptr *p, u32 start, u32 end
return 0;
}
-__bpf_kfunc bool bpf_dynptr_is_null(const struct bpf_dynptr *p)
-{
- struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
-
- return !ptr->data;
-}
-
-__bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p)
-{
- struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
-
- if (!ptr->data)
- return false;
-
- return __bpf_dynptr_is_rdonly(ptr);
-}
-
-__bpf_kfunc __u32 bpf_dynptr_size(const struct bpf_dynptr *p)
-{
- struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
-
- if (!ptr->data)
- return -EINVAL;
-
- return __bpf_dynptr_size(ptr);
-}
-
__bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p,
struct bpf_dynptr *clone__uninit)
{
diff --git a/kernel/bpf/inlinable_kfuncs.c b/kernel/bpf/inlinable_kfuncs.c
index 7b7dc05fa1a4..aeb3e3f209f7 100644
--- a/kernel/bpf/inlinable_kfuncs.c
+++ b/kernel/bpf/inlinable_kfuncs.c
@@ -1 +1,113 @@
// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/bpf.h>
+#include <linux/btf.h>
+#include <linux/skbuff.h>
+#include <linux/filter.h>
+
+__bpf_kfunc bool bpf_dynptr_is_null(const struct bpf_dynptr *p);
+__bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p);
+__bpf_kfunc __u32 bpf_dynptr_size(const struct bpf_dynptr *p);
+__bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr *p, u32 offset,
+ void *buffer__opt, u32 buffer__szk);
+
+__bpf_kfunc bool bpf_dynptr_is_null(const struct bpf_dynptr *p)
+{
+ struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+
+ return !ptr->data;
+}
+
+__bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p)
+{
+ struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+
+ if (!ptr->data)
+ return false;
+
+ return __bpf_dynptr_is_rdonly(ptr);
+}
+
+__bpf_kfunc __u32 bpf_dynptr_size(const struct bpf_dynptr *p)
+{
+ struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+
+ if (!ptr->data)
+ return -EINVAL;
+
+ return __bpf_dynptr_size(ptr);
+}
+
+/**
+ * bpf_dynptr_slice() - Obtain a read-only pointer to the dynptr data.
+ * @p: The dynptr whose data slice to retrieve
+ * @offset: Offset into the dynptr
+ * @buffer__opt: User-provided buffer to copy contents into. May be NULL
+ * @buffer__szk: Size (in bytes) of the buffer if present. This is the
+ * length of the requested slice. This must be a constant.
+ *
+ * For non-skb and non-xdp type dynptrs, there is no difference between
+ * bpf_dynptr_slice and bpf_dynptr_data.
+ *
+ * If buffer__opt is NULL, the call will fail if buffer_opt was needed.
+ *
+ * If the intention is to write to the data slice, please use
+ * bpf_dynptr_slice_rdwr.
+ *
+ * The user must check that the returned pointer is not null before using it.
+ *
+ * Please note that in the case of skb and xdp dynptrs, bpf_dynptr_slice
+ * does not change the underlying packet data pointers, so a call to
+ * bpf_dynptr_slice will not invalidate any ctx->data/data_end pointers in
+ * the bpf program.
+ *
+ * Return: NULL if the call failed (eg invalid dynptr), pointer to a read-only
+ * data slice (can be either direct pointer to the data or a pointer to the user
+ * provided buffer, with its contents containing the data, if unable to obtain
+ * direct pointer)
+ */
+__bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr *p, u32 offset,
+ void *buffer__opt, u32 buffer__szk)
+{
+ const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+ enum bpf_dynptr_type type;
+ u32 len = buffer__szk;
+ int err;
+
+ if (!ptr->data)
+ return NULL;
+
+ err = bpf_dynptr_check_off_len(ptr, offset, len);
+ if (err)
+ return NULL;
+
+ type = bpf_dynptr_get_type(ptr);
+
+ switch (type) {
+ case BPF_DYNPTR_TYPE_LOCAL:
+ case BPF_DYNPTR_TYPE_RINGBUF:
+ return ptr->data + ptr->offset + offset;
+ case BPF_DYNPTR_TYPE_SKB:
+ if (buffer__opt)
+ return skb_header_pointer(ptr->data, ptr->offset + offset, len, buffer__opt);
+ else
+ return skb_pointer_if_linear(ptr->data, ptr->offset + offset, len);
+ case BPF_DYNPTR_TYPE_XDP:
+ {
+ void *xdp_ptr = bpf_xdp_pointer(ptr->data, ptr->offset + offset, len);
+ if (!IS_ERR_OR_NULL(xdp_ptr))
+ return xdp_ptr;
+
+ if (!buffer__opt)
+ return NULL;
+ bpf_xdp_copy_buf(ptr->data, ptr->offset + offset, buffer__opt, len, false);
+ return buffer__opt;
+ }
+ default:
+ // TODO: can't handle inline assembly inside this when compiling to BPF
+#ifndef __FOR_BPF
+ WARN_ONCE(true, "unknown dynptr type %d\n", type);
+#endif
+ return NULL;
+ }
+}
--
2.47.0
