When instruction patching (addition or removal) occurs, the fdtab
attached to each subprog, and the program counter in its descriptors
will be out of sync wrt relative position in the program. To fix this,
we need to adjust the pc, free any unneeded fdtab and descriptors, and
ensure the entries correspond to the correct instruction offsets.
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@xxxxxxxxx>
---
kernel/bpf/verifier.c | 50 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 27233c308d83..e5b1db1db679 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -18737,6 +18737,23 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len
}
}
+static void adjust_subprog_frame_descs(struct bpf_verifier_env *env, u32 off, u32 len)
+{
+ if (len == 1)
+ return;
+ for (int i = 0; i <= env->subprog_cnt; i++) {
+ struct bpf_exception_frame_desc_tab *fdtab = subprog_info(env, i)->fdtab;
+
+ if (!fdtab)
+ continue;
+ for (int j = 0; j < fdtab->cnt; j++) {
+ if (fdtab->desc[j]->pc <= off)
+ continue;
+ fdtab->desc[j]->pc += len - 1;
+ }
+ }
+}
+
static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
{
struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
@@ -18775,6 +18792,7 @@ static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 of
}
adjust_insn_aux_data(env, new_data, new_prog, off, len);
adjust_subprog_starts(env, off, len);
+ adjust_subprog_frame_descs(env, off, len);
adjust_poke_descs(new_prog, off, len);
return new_prog;
}
@@ -18805,6 +18823,10 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
/* move fake 'exit' subprog as well */
move = env->subprog_cnt + 1 - j;
+ /* Free fdtab for subprog_info that we are going to destroy. */
+ for (int k = i; k < j; k++)
+ bpf_exception_frame_desc_tab_free(env->subprog_info[k].fdtab);
+
memmove(env->subprog_info + i,
env->subprog_info + j,
sizeof(*env->subprog_info) * move);
@@ -18835,6 +18857,30 @@ static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
return 0;
}
+static int adjust_subprog_frame_descs_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt)
+{
+ for (int i = 0; i < env->subprog_cnt; i++) {
+ struct bpf_exception_frame_desc_tab *fdtab = subprog_info(env, i)->fdtab;
+
+ if (!fdtab)
+ continue;
+ for (int j = 0; j < fdtab->cnt; j++) {
+ /* Part of a subprog_info whose instructions were removed partially, but the fdtab remained. */
+ if (fdtab->desc[j]->pc >= off && fdtab->desc[j]->pc < off + cnt) {
+ void *p = fdtab->desc[j];
+ if (j < fdtab->cnt - 1)
+ memmove(fdtab->desc + j, fdtab->desc + j + 1, sizeof(fdtab->desc[0]) * (fdtab->cnt - j - 1));
+ kfree(p);
+ fdtab->cnt--;
+ j--;
+ }
+ if (fdtab->desc[j]->pc >= off + cnt)
+ fdtab->desc[j]->pc -= cnt;
+ }
+ }
+ return 0;
+}
+
static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
u32 cnt)
{
@@ -18916,6 +18962,10 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
if (err)
return err;
+ err = adjust_subprog_frame_descs_after_remove(env, off, cnt);
+ if (err)
+ return err;
+
err = bpf_adj_linfo_after_remove(env, off, cnt);
if (err)
return err;
--
2.40.1
