On 04/16, Alexei Starovoitov wrote:
On Thu, Apr 14, 2022 at 9:12 AM Stanislav Fomichev <sdf@xxxxxxxxxx> wrote:
> +static int
> +bpf_prog_run_array_cg_flags(const struct cgroup_bpf *cgrp,
> + enum cgroup_bpf_attach_type atype,
> + const void *ctx, bpf_prog_run_fn run_prog,
> + int retval, u32 *ret_flags)
> +{
> + const struct bpf_prog_array_item *item;
> + const struct bpf_prog *prog;
> + const struct bpf_prog_array *array;
> + struct bpf_run_ctx *old_run_ctx;
> + struct bpf_cg_run_ctx run_ctx;
> + u32 func_ret;
> +
> + run_ctx.retval = retval;
> + migrate_disable();
> + rcu_read_lock();
> + array = rcu_dereference(cgrp->effective[atype]);
> + item = &array->items[0];
> + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
> + while ((prog = READ_ONCE(item->prog))) {
> + run_ctx.prog_item = item;
> + func_ret = run_prog(prog, ctx);
...
> + ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
> &ctx, bpf_prog_run, retval);
Did you check the asm that bpf_prog_run gets inlined
after being passed as a pointer to a function?
Crossing fingers... I suspect not every compiler can do that :(
De-virtualization optimization used to be tricky.
No, I didn't, but looking at it right now, both gcc and clang
seem to be doing inlining all way up to bpf_dispatcher_nop_func.
clang:
0000000000001750 <__cgroup_bpf_run_filter_sock_addr>:
__cgroup_bpf_run_filter_sock_addr():
./kernel/bpf/cgroup.c:1226
int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
struct sockaddr *uaddr,
enum cgroup_bpf_attach_type atype,
void *t_ctx,
u32 *flags)
{
...
./include/linux/filter.h:628
ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
1980: 49 8d 75 48 lea 0x48(%r13),%rsi
bpf_dispatcher_nop_func():
./include/linux/bpf.h:804
return bpf_func(ctx, insnsi);
1984: 4c 89 f7 mov %r14,%rdi
1987: 41 ff 55 30 call *0x30(%r13)
198b: 89 c3 mov %eax,%ebx
gcc (w/retpoline):
0000000000001110 <__cgroup_bpf_run_filter_sock_addr>:
__cgroup_bpf_run_filter_sock_addr():
kernel/bpf/cgroup.c:1226
{
...
./include/linux/filter.h:628
ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
11c5: 49 8d 75 48 lea 0x48(%r13),%rsi
bpf_dispatcher_nop_func():
./include/linux/bpf.h:804
11c9: 48 8d 7c 24 10 lea 0x10(%rsp),%rdi
11ce: e8 00 00 00 00 call 11d3
<__cgroup_bpf_run_filter_sock_addr+0xc3>
11cf: R_X86_64_PLT32 __x86_indirect_thunk_rax-0x4
11d3: 89 c3 mov %eax,%ebx
