The next patch adds the ability to create sleepable bpf iterator programs.
This changes `bpf_iter_run_prog` to use the appropriate synchronization for
sleepable bpf programs. With sleepable bpf iterator programs, we can no
longer use `rcu_read_lock()` and must use `rcu_read_lock_trace()` instead
to protect the bpf program.
Signed-off-by: Kenny Yu <kennyyu@xxxxxx>
---
kernel/bpf/bpf_iter.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index b7aef5b3416d..d814ca6454af 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -5,6 +5,7 @@
#include <linux/anon_inodes.h>
#include <linux/filter.h>
#include <linux/bpf.h>
+#include <linux/rcupdate_trace.h>
struct bpf_iter_target_info {
struct list_head list;
@@ -684,11 +685,20 @@ int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
{
int ret;
- rcu_read_lock();
- migrate_disable();
+ if (prog->aux->sleepable) {
+ rcu_read_lock_trace();
+ migrate_disable();
+ might_fault();
+ } else {
+ rcu_read_lock();
+ migrate_disable();
+ }
ret = bpf_prog_run(prog, ctx);
migrate_enable();
- rcu_read_unlock();
+ if (prog->aux->sleepable)
+ rcu_read_unlock_trace();
+ else
+ rcu_read_unlock();
/* bpf program can only return 0 or 1:
* 0 : okay
--
2.30.2
