From: Hou Tao <houtao1@xxxxxxxxxx>
Considering most 64-bit JIT backends support atomic_xchg() and the
implementation of xchg() is the same as atomic_xchg() on these 64-bits
architectures, inline bpf_kptr_xchg() by converting the calling of
bpf_kptr_xchg() into an atomic_xchg() instruction.
As a precaution, defining a weak function bpf_jit_supports_ptr_xchg()
to state whether such conversion is safe and only inlining under 64-bit
host.
Signed-off-by: Hou Tao <houtao1@xxxxxxxxxx>
---
include/linux/filter.h | 1 +
kernel/bpf/core.c | 10 ++++++++++
kernel/bpf/verifier.c | 17 +++++++++++++++++
3 files changed, 28 insertions(+)
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 12d907f17d36..fee070b9826e 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -955,6 +955,7 @@ bool bpf_jit_supports_subprog_tailcalls(void);
bool bpf_jit_supports_kfunc_call(void);
bool bpf_jit_supports_far_kfunc_call(void);
bool bpf_jit_supports_exceptions(void);
+bool bpf_jit_supports_ptr_xchg(void);
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
bool bpf_helper_changes_pkt_data(void *func);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 5aa6863ac33b..b64885827f90 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -2922,6 +2922,16 @@ bool __weak bpf_jit_supports_far_kfunc_call(void)
return false;
}
+/* Return TRUE if the JIT backend satisfies the following two conditions:
+ * 1) JIT backend supports atomic_xchg() on pointer-sized words.
+ * 2) Under the specific arch, the implementation of xchg() is the same
+ * as atomic_xchg() on pointer-sized words.
+ */
+bool __weak bpf_jit_supports_ptr_xchg(void)
+{
+ return false;
+}
+
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call
* skb_copy_bits(), so provide a weak definition of it for NET-less config.
*/
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9456ee0ad129..7814c4f7576e 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -19668,6 +19668,23 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
continue;
}
+ /* Implement bpf_kptr_xchg inline */
+ if (prog->jit_requested && BITS_PER_LONG == 64 &&
+ insn->imm == BPF_FUNC_kptr_xchg &&
+ bpf_jit_supports_ptr_xchg()) {
+ insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
+ insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
+ cnt = 2;
+
+ new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
+ if (!new_prog)
+ return -ENOMEM;
+
+ delta += cnt - 1;
+ env->prog = prog = new_prog;
+ insn = new_prog->insnsi + i + delta;
+ continue;
+ }
patch_call_imm:
fn = env->ops->get_func_proto(insn->imm, env->prog);
/* all functions that have prototype and verifier allowed
--
2.29.2
