Follow-up patches use add_hidden_subprog() to inject inlinable kfunc
bodies into bpf program as subprograms. At the moment only one hidden
subprogram is allowed, as bpf_verifier_env->subprog_info is allocated
in advance as array of fixed size. This patch removes the limitation
by using dynamic memory allocation for this array.
Signed-off-by: Eduard Zingerman <eddyz87@xxxxxxxxx>
---
include/linux/bpf_verifier.h | 3 ++-
kernel/bpf/verifier.c | 29 ++++++++++++++++++++++-------
2 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index ed4eacfd4db7..b683dc3ede4a 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -738,7 +738,7 @@ struct bpf_verifier_env {
struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */
const struct bpf_line_info *prev_linfo;
struct bpf_verifier_log log;
- struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */
+ struct bpf_subprog_info *subprog_info;
union {
struct bpf_idmap idmap_scratch;
struct bpf_idset idset_scratch;
@@ -751,6 +751,7 @@ struct bpf_verifier_env {
struct backtrack_state bt;
struct bpf_jmp_history_entry *cur_hist_ent;
u32 pass_cnt; /* number of times do_check() was called */
+ u32 subprog_cap;
u32 subprog_cnt;
/* number of instructions analyzed by the verifier */
u32 prev_insn_processed, insn_processed;
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index b86308896358..d4ea7fd8a967 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19419,7 +19419,7 @@ static int adjust_jmp_off(struct bpf_prog *prog, u32 tgt_idx, u32 delta)
static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
u32 off, u32 cnt)
{
- int i, j;
+ int i, j, first_hidden = env->subprog_cnt - env->hidden_subprog_cnt;
/* find first prog starting at or after off (first to remove) */
for (i = 0; i < env->subprog_cnt; i++)
@@ -19446,6 +19446,8 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
env->subprog_info + j,
sizeof(*env->subprog_info) * move);
env->subprog_cnt -= j - i;
+ if (first_hidden <= j - 1)
+ env->hidden_subprog_cnt -= j - first_hidden;
/* remove func_info */
if (aux->func_info) {
@@ -21215,15 +21217,20 @@ static int resolve_kfunc_calls(struct bpf_verifier_env *env)
/* The function requires that first instruction in 'patch' is insnsi[prog->len - 1] */
static int add_hidden_subprog(struct bpf_verifier_env *env, struct bpf_insn *patch, int len)
{
- struct bpf_subprog_info *info = env->subprog_info;
+ struct bpf_subprog_info *info, *tmp;
int cnt = env->subprog_cnt;
struct bpf_prog *prog;
- /* We only reserve one slot for hidden subprogs in subprog_info. */
- if (env->hidden_subprog_cnt) {
- verbose(env, "verifier internal error: only one hidden subprog supported\n");
- return -EFAULT;
+ if (cnt == env->subprog_cap) {
+ env->subprog_cap *= 2;
+ tmp = vrealloc(env->subprog_info,
+ array_size(sizeof(*env->subprog_info), env->subprog_cap + 1),
+ GFP_KERNEL | __GFP_ZERO);
+ if (!tmp)
+ return -ENOMEM;
+ env->subprog_info = tmp;
}
+ info = env->subprog_info;
/* We're not patching any existing instruction, just appending the new
* ones for the hidden subprog. Hence all of the adjustment operations
* in bpf_patch_insn_data are no-ops.
@@ -23122,6 +23129,13 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
ret = -ENOMEM;
if (!env->insn_aux_data)
goto err_free_env;
+ env->subprog_cap = BPF_MAX_SUBPROGS;
+ env->subprog_info = vzalloc(array_size(sizeof(*env->subprog_info),
+ env->subprog_cap + 1 /* max + 1 for the fake subprog */));
+ if (!env->subprog_info) {
+ ret = -ENOMEM;
+ goto err_free_env;
+ }
for (i = 0; i < len; i++)
env->insn_aux_data[i].orig_idx = i;
env->prog = *prog;
@@ -23353,8 +23367,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
err_unlock:
if (!is_priv)
mutex_unlock(&bpf_verifier_lock);
- vfree(env->insn_aux_data);
err_free_env:
+ vfree(env->subprog_info);
+ vfree(env->insn_aux_data);
kvfree(env);
return ret;
}
--
2.47.0
