Add a new bpf system call, BPF_STATIC_BRANCH_UPDATE, which allows users to update static branches in BPF. Namely, this system call is executed as bpf(BPF_STATIC_BRANCH_UPDATE, attrs={prog_fd, insn_off, on}) where prog_fd points to a BPF program, insn_off is an _xlated_ offset in this program, on is a boolean value to set this branch on or off. The instruction at insn_off must be a JA with SRC_REG or'ed with BPF_STATIC_BRANCH_JA and, optionally, with BPF_STATIC_BRANCH_INVERSE. To implement this for a particular architecture, re-define the weak bpf_arch_poke_static_branch() function in the corresponding bpf_jit_comp.c This patch adds x86 implementation. Signed-off-by: Anton Protopopov <aspsk@xxxxxxxxxxxxx> --- arch/x86/net/bpf_jit_comp.c | 40 +++++++++++++++++++++++ include/linux/bpf.h | 2 ++ include/linux/filter.h | 1 + include/uapi/linux/bpf.h | 7 ++++ kernel/bpf/core.c | 5 +++ kernel/bpf/syscall.c | 60 ++++++++++++++++++++++++++++++++++ tools/include/uapi/linux/bpf.h | 7 ++++ 7 files changed, 122 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index b291b5c79d26..2090713e4126 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2099,8 +2099,17 @@ st: if (is_imm8(insn->off)) if (func_idx) off += bpf_prog->aux->func_info[func_idx].insn_off; + bpf_prog->aux->xlated_to_jit[off].ip = image + proglen; bpf_prog->aux->xlated_to_jit[off].off = proglen; bpf_prog->aux->xlated_to_jit[off].len = ilen; + + /* + * Save the offset so that it can later be accessed + * by the bpf(BPF_STATIC_BRANCH_UPDATE) syscall + */ + if (insn->code == (BPF_JMP | BPF_JA) || + insn->code == (BPF_JMP32 | BPF_JA)) + bpf_prog->aux->xlated_to_jit[off].jmp_offset = jmp_offset; } } proglen += ilen; @@ -3276,3 +3285,34 @@ bool bpf_jit_supports_ptr_xchg(void) { return true; } + +int bpf_arch_poke_static_branch(struct bpf_prog *prog, + u32 insn_off, + bool on) +{ + int jmp_offset = prog->aux->xlated_to_jit[insn_off].jmp_offset; + u32 len = prog->aux->xlated_to_jit[insn_off].len; + u8 op[5]; + + if (WARN_ON_ONCE(is_imm8(jmp_offset) && len != 2)) + return -EINVAL; + + if (WARN_ON_ONCE(!is_imm8(jmp_offset) && len != 5)) + return -EINVAL; + + if (on) { + if (len == 2) { + op[0] = 0xEB; + op[1] = jmp_offset; + } else { + op[0] = 0xE9; + memcpy(&op[1], &jmp_offset, 4); + } + } else { + memcpy(op, x86_nops[len], len); + } + + text_poke_bp(prog->aux->xlated_to_jit[insn_off].ip, op, len, NULL); + + return 0; +} diff --git a/include/linux/bpf.h b/include/linux/bpf.h index bdd6be718e82..1363b1fc8c09 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1528,8 +1528,10 @@ struct bpf_prog_aux { * instructions, if allocated */ struct { + void *ip; /* the address of the jitted insn */ u32 off; /* local offset in the jitted code */ u32 len; /* the total len of generated jit code */ + u32 jmp_offset; /* jitted jump offset for BPF_JA insns */ } *xlated_to_jit; }; diff --git a/include/linux/filter.h b/include/linux/filter.h index fee070b9826e..0dad44fa3af2 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -957,6 +957,7 @@ bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); +int bpf_arch_poke_static_branch(struct bpf_prog *prog, u32 off, bool on); bool bpf_helper_changes_pkt_data(void *func); static inline bool bpf_dump_raw_ok(const struct cred *cred) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index aca5ed065731..8aafb0eddd1c 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -932,6 +932,7 @@ enum bpf_cmd { BPF_LINK_DETACH, BPF_PROG_BIND_MAP, BPF_TOKEN_CREATE, + BPF_STATIC_BRANCH_UPDATE, __MAX_BPF_CMD, }; @@ -1787,6 +1788,12 @@ union bpf_attr { __u32 bpffs_fd; } token_create; + struct { /* struct used by BPF_STATIC_BRANCH_UPDATE command */ + __u32 prog_fd; + __u32 insn_off; + __u32 on; + } static_branch; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 8e99c1563a7f..fec185354ea3 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3042,6 +3042,11 @@ static int __init bpf_global_ma_init(void) late_initcall(bpf_global_ma_init); #endif +int __weak bpf_arch_poke_static_branch(struct bpf_prog *prog, u32 off, bool on) +{ + return -EOPNOTSUPP; +} + DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); EXPORT_SYMBOL(bpf_stats_enabled_key); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 36b8fdcfba75..9e2e12a0bdfe 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1540,6 +1540,63 @@ static int map_lookup_elem(union bpf_attr *attr) return err; } +static int parse_static_branch_insn(struct bpf_insn *insn, bool *inverse) +{ + __u8 code = insn->code; + + if (code != (BPF_JMP | BPF_JA) && code != (BPF_JMP32 | BPF_JA)) + return -EINVAL; + + if (insn->src_reg & ~BPF_STATIC_BRANCH_MASK) + return -EINVAL; + + if (!(insn->src_reg & BPF_STATIC_BRANCH_JA)) + return -EINVAL; + + if (insn->dst_reg) + return -EINVAL; + + *inverse = !(insn->src_reg & BPF_STATIC_BRANCH_NOP); + + return 0; +} + +#define BPF_STATIC_BRANCH_UPDATE_LAST_FIELD static_branch.on + +static int bpf_static_branch_update(union bpf_attr *attr) +{ + bool on = attr->static_branch.on & 1; + struct bpf_prog *prog; + u32 insn_off; + bool inverse; + int ret; + + if (CHECK_ATTR(BPF_STATIC_BRANCH_UPDATE)) + return -EINVAL; + + if (attr->static_branch.on & ~1) + return -EINVAL; + + prog = bpf_prog_get(attr->static_branch.prog_fd); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + insn_off = attr->static_branch.insn_off; + if (insn_off >= prog->len) { + ret = -ERANGE; + goto put_prog; + } + + ret = parse_static_branch_insn(&prog->insnsi[insn_off], &inverse); + if (ret) + goto put_prog; + + ret = bpf_arch_poke_static_branch(prog, insn_off, on ^ inverse); + +put_prog: + bpf_prog_put(prog); + return ret; +} #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags @@ -5694,6 +5751,9 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) case BPF_MAP_DELETE_BATCH: err = bpf_map_do_batch(&attr, uattr.user, BPF_MAP_DELETE_BATCH); break; + case BPF_STATIC_BRANCH_UPDATE: + err = bpf_static_branch_update(&attr); + break; case BPF_LINK_CREATE: err = link_create(&attr, uattr); break; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index aca5ed065731..8aafb0eddd1c 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -932,6 +932,7 @@ enum bpf_cmd { BPF_LINK_DETACH, BPF_PROG_BIND_MAP, BPF_TOKEN_CREATE, + BPF_STATIC_BRANCH_UPDATE, __MAX_BPF_CMD, }; @@ -1787,6 +1788,12 @@ union bpf_attr { __u32 bpffs_fd; } token_create; + struct { /* struct used by BPF_STATIC_BRANCH_UPDATE command */ + __u32 prog_fd; + __u32 insn_off; + __u32 on; + } static_branch; + } __attribute__((aligned(8))); /* The description below is an attempt at providing documentation to eBPF -- 2.34.1