These helpers are equivalent to bpf_spin_{lock,unlock}, but the verifier
doesn't try to enforce that no helper calls occur when there's an active
spin lock.
[ TODO: Currently the verifier doesn't do _anything_ spinlock related
when it sees one of these, including setting active_spin_lock. This is
probably too lenient. Also, EXPORT_SYMBOL for internal lock helpers
might not be the best code structure. ]
Future patches will add enforcement of "rbtree helpers must always be
called when lock is held" constraint.
Signed-off-by: Dave Marchevsky <davemarchevsky@xxxxxx>
---
include/uapi/linux/bpf.h | 20 ++++++++++++++++++++
kernel/bpf/helpers.c | 12 ++++++++++--
kernel/bpf/rbtree.c | 29 +++++++++++++++++++++++++++++
kernel/bpf/verifier.c | 2 ++
tools/include/uapi/linux/bpf.h | 20 ++++++++++++++++++++
5 files changed, 81 insertions(+), 2 deletions(-)
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 06d71207de0b..f4c615fbf64f 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -5415,6 +5415,24 @@ union bpf_attr {
*
* Return
* Ptr to lock
+ *
+ * void *bpf_rbtree_lock(struct bpf_spin_lock *lock)
+ * Description
+ * Like bpf_spin_lock helper, but use separate helper for now
+ * as we don't want this helper to have special meaning to the verifier
+ * so that we can do rbtree helper calls between rbtree_lock/unlock
+ *
+ * Return
+ * 0
+ *
+ * void *bpf_rbtree_unlock(struct bpf_spin_lock *lock)
+ * Description
+ * Like bpf_spin_unlock helper, but use separate helper for now
+ * as we don't want this helper to have special meaning to the verifier
+ * so that we can do rbtree helper calls between rbtree_lock/unlock
+ *
+ * Return
+ * 0
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -5632,6 +5650,8 @@ union bpf_attr {
FN(rbtree_remove), \
FN(rbtree_free_node), \
FN(rbtree_get_lock), \
+ FN(rbtree_lock), \
+ FN(rbtree_unlock), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index ae974d0aa70d..0ca5fed1013b 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -316,7 +316,7 @@ static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
static DEFINE_PER_CPU(unsigned long, irqsave_flags);
-static inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock)
+inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock)
{
unsigned long flags;
@@ -324,6 +324,7 @@ static inline void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock)
__bpf_spin_lock(lock);
__this_cpu_write(irqsave_flags, flags);
}
+EXPORT_SYMBOL(__bpf_spin_lock_irqsave);
notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
{
@@ -338,7 +339,7 @@ const struct bpf_func_proto bpf_spin_lock_proto = {
.arg1_type = ARG_PTR_TO_SPIN_LOCK,
};
-static inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock)
+inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock)
{
unsigned long flags;
@@ -346,6 +347,7 @@ static inline void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock)
__bpf_spin_unlock(lock);
local_irq_restore(flags);
}
+EXPORT_SYMBOL(__bpf_spin_unlock_irqrestore);
notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
{
@@ -1604,6 +1606,8 @@ const struct bpf_func_proto bpf_rbtree_find_proto __weak;
const struct bpf_func_proto bpf_rbtree_remove_proto __weak;
const struct bpf_func_proto bpf_rbtree_free_node_proto __weak;
const struct bpf_func_proto bpf_rbtree_get_lock_proto __weak;
+const struct bpf_func_proto bpf_rbtree_lock_proto __weak;
+const struct bpf_func_proto bpf_rbtree_unlock_proto __weak;
const struct bpf_func_proto *
bpf_base_func_proto(enum bpf_func_id func_id)
@@ -1707,6 +1711,10 @@ bpf_base_func_proto(enum bpf_func_id func_id)
return &bpf_rbtree_free_node_proto;
case BPF_FUNC_rbtree_get_lock:
return &bpf_rbtree_get_lock_proto;
+ case BPF_FUNC_rbtree_lock:
+ return &bpf_rbtree_lock_proto;
+ case BPF_FUNC_rbtree_unlock:
+ return &bpf_rbtree_unlock_proto;
default:
break;
}
diff --git a/kernel/bpf/rbtree.c b/kernel/bpf/rbtree.c
index 0cc495b7cb26..641821ee1a7f 100644
--- a/kernel/bpf/rbtree.c
+++ b/kernel/bpf/rbtree.c
@@ -262,6 +262,35 @@ const struct bpf_func_proto bpf_rbtree_get_lock_proto = {
.arg1_type = ARG_CONST_MAP_PTR,
};
+extern void __bpf_spin_unlock_irqrestore(struct bpf_spin_lock *lock);
+extern void __bpf_spin_lock_irqsave(struct bpf_spin_lock *lock);
+
+BPF_CALL_1(bpf_rbtree_lock, void *, lock)
+{
+ __bpf_spin_lock_irqsave((struct bpf_spin_lock *)lock);
+ return 0;
+}
+
+const struct bpf_func_proto bpf_rbtree_lock_proto = {
+ .func = bpf_rbtree_lock,
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_SPIN_LOCK,
+};
+
+BPF_CALL_1(bpf_rbtree_unlock, void *, lock)
+{
+ __bpf_spin_unlock_irqrestore((struct bpf_spin_lock *)lock);
+ return 0;
+}
+
+const struct bpf_func_proto bpf_rbtree_unlock_proto = {
+ .func = bpf_rbtree_unlock,
+ .gpl_only = true,
+ .ret_type = RET_INTEGER,
+ .arg1_type = ARG_PTR_TO_SPIN_LOCK,
+};
+
BTF_ID_LIST_SINGLE(bpf_rbtree_map_btf_ids, struct, bpf_rbtree)
const struct bpf_map_ops rbtree_map_ops = {
.map_meta_equal = bpf_map_meta_equal,
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 0a2e958ddca8..b9e5d87fe323 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -6057,6 +6057,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
} else if (meta->func_id == BPF_FUNC_spin_unlock) {
if (process_spin_lock(env, regno, false))
return -EACCES;
+ } else if (meta->func_id == BPF_FUNC_rbtree_lock ||
+ meta->func_id == BPF_FUNC_rbtree_unlock) { // Do nothing for now
} else {
verbose(env, "verifier internal error\n");
return -EFAULT;
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 06d71207de0b..f4c615fbf64f 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -5415,6 +5415,24 @@ union bpf_attr {
*
* Return
* Ptr to lock
+ *
+ * void *bpf_rbtree_lock(struct bpf_spin_lock *lock)
+ * Description
+ * Like bpf_spin_lock helper, but use separate helper for now
+ * as we don't want this helper to have special meaning to the verifier
+ * so that we can do rbtree helper calls between rbtree_lock/unlock
+ *
+ * Return
+ * 0
+ *
+ * void *bpf_rbtree_unlock(struct bpf_spin_lock *lock)
+ * Description
+ * Like bpf_spin_unlock helper, but use separate helper for now
+ * as we don't want this helper to have special meaning to the verifier
+ * so that we can do rbtree helper calls between rbtree_lock/unlock
+ *
+ * Return
+ * 0
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -5632,6 +5650,8 @@ union bpf_attr {
FN(rbtree_remove), \
FN(rbtree_free_node), \
FN(rbtree_get_lock), \
+ FN(rbtree_lock), \
+ FN(rbtree_unlock), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
--
2.30.2
