On 04/18, Alexei Starovoitov wrote:
On Mon, Apr 18, 2022 at 9:50 AM <sdf@xxxxxxxxxx> wrote:
>
> On 04/16, Alexei Starovoitov wrote:
> > On Thu, Apr 14, 2022 at 9:12 AM Stanislav Fomichev <sdf@xxxxxxxxxx>
wrote:
> > > +static int
> > > +bpf_prog_run_array_cg_flags(const struct cgroup_bpf *cgrp,
> > > + enum cgroup_bpf_attach_type atype,
> > > + const void *ctx, bpf_prog_run_fn
run_prog,
> > > + int retval, u32 *ret_flags)
> > > +{
> > > + const struct bpf_prog_array_item *item;
> > > + const struct bpf_prog *prog;
> > > + const struct bpf_prog_array *array;
> > > + struct bpf_run_ctx *old_run_ctx;
> > > + struct bpf_cg_run_ctx run_ctx;
> > > + u32 func_ret;
> > > +
> > > + run_ctx.retval = retval;
> > > + migrate_disable();
> > > + rcu_read_lock();
> > > + array = rcu_dereference(cgrp->effective[atype]);
> > > + item = &array->items[0];
> > > + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
> > > + while ((prog = READ_ONCE(item->prog))) {
> > > + run_ctx.prog_item = item;
> > > + func_ret = run_prog(prog, ctx);
> > ...
> > > + ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
> > > &ctx, bpf_prog_run, retval);
>
> > Did you check the asm that bpf_prog_run gets inlined
> > after being passed as a pointer to a function?
> > Crossing fingers... I suspect not every compiler can do that :(
> > De-virtualization optimization used to be tricky.
>
> No, I didn't, but looking at it right now, both gcc and clang
> seem to be doing inlining all way up to bpf_dispatcher_nop_func.
>
> clang:
>
> 0000000000001750 <__cgroup_bpf_run_filter_sock_addr>:
> __cgroup_bpf_run_filter_sock_addr():
> ./kernel/bpf/cgroup.c:1226
> int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
> struct sockaddr *uaddr,
> enum cgroup_bpf_attach_type atype,
> void *t_ctx,
> u32 *flags)
> {
>
> ...
>
> ./include/linux/filter.h:628
> ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
> 1980: 49 8d 75 48 lea 0x48(%r13),%rsi
> bpf_dispatcher_nop_func():
> ./include/linux/bpf.h:804
> return bpf_func(ctx, insnsi);
> 1984: 4c 89 f7 mov %r14,%rdi
> 1987: 41 ff 55 30 call *0x30(%r13)
> 198b: 89 c3 mov %eax,%ebx
>
> gcc (w/retpoline):
>
> 0000000000001110 <__cgroup_bpf_run_filter_sock_addr>:
> __cgroup_bpf_run_filter_sock_addr():
> kernel/bpf/cgroup.c:1226
> {
>
> ...
>
> ./include/linux/filter.h:628
> ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
> 11c5: 49 8d 75 48 lea 0x48(%r13),%rsi
> bpf_dispatcher_nop_func():
> ./include/linux/bpf.h:804
> 11c9: 48 8d 7c 24 10 lea 0x10(%rsp),%rdi
> 11ce: e8 00 00 00 00 call 11d3
> <__cgroup_bpf_run_filter_sock_addr+0xc3>
> 11cf: R_X86_64_PLT32
__x86_indirect_thunk_rax-0x4
> 11d3: 89 c3 mov %eax,%ebx
Hmm. I'm not sure how you've got this asm.
Here is what I see with gcc 8 and gcc 10:
bpf_prog_run_array_cg:
...
movq %rcx, %r12 # run_prog, run_prog
...
# ../kernel/bpf/cgroup.c:77: run_ctx.prog_item = item;
movq %rbx, (%rsp) # item, run_ctx.prog_item
# ../kernel/bpf/cgroup.c:78: if (!run_prog(prog, ctx) &&
!IS_ERR_VALUE((long)run_ctx.retval))
movq %rbp, %rsi # ctx,
call *%r12 # run_prog
__cgroup_bpf_run_filter_sk:
movq $bpf_prog_run, %rcx #,
# ../kernel/bpf/cgroup.c:1202: return
bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0);
leaq 1520(%rax), %rdi #, tmp92
# ../kernel/bpf/cgroup.c:1202: return
bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0);
jmp bpf_prog_run_array_cg #
This is without kasan, lockdep and all debug configs are off.
So the generated code is pretty bad as I predicted :(
So I'm afraid this approach is no go.
I've retested again and it still unrolls it for me on gcc 11 :-/
Anyway, I guess we have two options:
1. Go back to defines.
2. Don't pass a ptr to func, but pass an enum which indicates whether
to use bpf_prog_run or __bpf_prog_run_save_cb. Seems like in this
case the compiler shouldn't have any trouble unwrapping it?
I'll prototype and send (2). If it won't work out we can always get back
to (1).
