Re: [PATCH bpf-next v2] bpf: move rcu lock management out of BPF_PROG_RUN routines

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Tue, Apr 19, 2022 at 9:20 AM Alexei Starovoitov
<alexei.starovoitov@xxxxxxxxx> wrote:
>
> On Tue, Apr 19, 2022 at 8:42 AM <sdf@xxxxxxxxxx> wrote:
> >
> > On 04/18, Alexei Starovoitov wrote:
> > > On Mon, Apr 18, 2022 at 9:50 AM <sdf@xxxxxxxxxx> wrote:
> > > >
> > > > On 04/16, Alexei Starovoitov wrote:
> > > > > On Thu, Apr 14, 2022 at 9:12 AM Stanislav Fomichev <sdf@xxxxxxxxxx>
> > > wrote:
> > > > > > +static int
> > > > > > +bpf_prog_run_array_cg_flags(const struct cgroup_bpf *cgrp,
> > > > > > +                           enum cgroup_bpf_attach_type atype,
> > > > > > +                           const void *ctx, bpf_prog_run_fn
> > > run_prog,
> > > > > > +                           int retval, u32 *ret_flags)
> > > > > > +{
> > > > > > +       const struct bpf_prog_array_item *item;
> > > > > > +       const struct bpf_prog *prog;
> > > > > > +       const struct bpf_prog_array *array;
> > > > > > +       struct bpf_run_ctx *old_run_ctx;
> > > > > > +       struct bpf_cg_run_ctx run_ctx;
> > > > > > +       u32 func_ret;
> > > > > > +
> > > > > > +       run_ctx.retval = retval;
> > > > > > +       migrate_disable();
> > > > > > +       rcu_read_lock();
> > > > > > +       array = rcu_dereference(cgrp->effective[atype]);
> > > > > > +       item = &array->items[0];
> > > > > > +       old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
> > > > > > +       while ((prog = READ_ONCE(item->prog))) {
> > > > > > +               run_ctx.prog_item = item;
> > > > > > +               func_ret = run_prog(prog, ctx);
> > > > > ...
> > > > > > +       ret = bpf_prog_run_array_cg(&cgrp->bpf, CGROUP_GETSOCKOPT,
> > > > > >                                     &ctx, bpf_prog_run, retval);
> > > >
> > > > > Did you check the asm that bpf_prog_run gets inlined
> > > > > after being passed as a pointer to a function?
> > > > > Crossing fingers... I suspect not every compiler can do that :(
> > > > > De-virtualization optimization used to be tricky.
> > > >
> > > > No, I didn't, but looking at it right now, both gcc and clang
> > > > seem to be doing inlining all way up to bpf_dispatcher_nop_func.
> > > >
> > > > clang:
> > > >
> > > >    0000000000001750 <__cgroup_bpf_run_filter_sock_addr>:
> > > >    __cgroup_bpf_run_filter_sock_addr():
> > > >    ./kernel/bpf/cgroup.c:1226
> > > >    int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
> > > >                                       struct sockaddr *uaddr,
> > > >                                       enum cgroup_bpf_attach_type atype,
> > > >                                       void *t_ctx,
> > > >                                       u32 *flags)
> > > >    {
> > > >
> > > >    ...
> > > >
> > > >    ./include/linux/filter.h:628
> > > >                 ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
> > > >        1980:    49 8d 75 48             lea    0x48(%r13),%rsi
> > > >    bpf_dispatcher_nop_func():
> > > >    ./include/linux/bpf.h:804
> > > >         return bpf_func(ctx, insnsi);
> > > >        1984:    4c 89 f7                mov    %r14,%rdi
> > > >        1987:    41 ff 55 30             call   *0x30(%r13)
> > > >        198b:    89 c3                   mov    %eax,%ebx
> > > >
> > > > gcc (w/retpoline):
> > > >
> > > >    0000000000001110 <__cgroup_bpf_run_filter_sock_addr>:
> > > >    __cgroup_bpf_run_filter_sock_addr():
> > > >    kernel/bpf/cgroup.c:1226
> > > >    {
> > > >
> > > >    ...
> > > >
> > > >    ./include/linux/filter.h:628
> > > >                 ret = dfunc(ctx, prog->insnsi, prog->bpf_func);
> > > >        11c5:    49 8d 75 48             lea    0x48(%r13),%rsi
> > > >    bpf_dispatcher_nop_func():
> > > >    ./include/linux/bpf.h:804
> > > >        11c9:    48 8d 7c 24 10          lea    0x10(%rsp),%rdi
> > > >        11ce:    e8 00 00 00 00          call   11d3
> > > > <__cgroup_bpf_run_filter_sock_addr+0xc3>
> > > >                         11cf: R_X86_64_PLT32
> > > __x86_indirect_thunk_rax-0x4
> > > >        11d3:    89 c3                   mov    %eax,%ebx
> >
> > > Hmm. I'm not sure how you've got this asm.
> > > Here is what I see with gcc 8 and gcc 10:
> > > bpf_prog_run_array_cg:
> > > ...
> > >          movq    %rcx, %r12      # run_prog, run_prog
> > > ...
> > > # ../kernel/bpf/cgroup.c:77:            run_ctx.prog_item = item;
> > >          movq    %rbx, (%rsp)    # item, run_ctx.prog_item
> > > # ../kernel/bpf/cgroup.c:78:            if (!run_prog(prog, ctx) &&
> > > !IS_ERR_VALUE((long)run_ctx.retval))
> > >          movq    %rbp, %rsi      # ctx,
> > >          call    *%r12   # run_prog
> >
> > > __cgroup_bpf_run_filter_sk:
> > >          movq    $bpf_prog_run, %rcx     #,
> > > # ../kernel/bpf/cgroup.c:1202:  return
> > > bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0);
> > >          leaq    1520(%rax), %rdi        #, tmp92
> > > # ../kernel/bpf/cgroup.c:1202:  return
> > > bpf_prog_run_array_cg(&cgrp->bpf, atype, sk, bpf_prog_run, 0);
> > >          jmp     bpf_prog_run_array_cg   #
> >
> > > This is without kasan, lockdep and all debug configs are off.
> >
> > > So the generated code is pretty bad as I predicted :(
> >
> > > So I'm afraid this approach is no go.
> >
> > I've retested again and it still unrolls it for me on gcc 11 :-/
> > Anyway, I guess we have two options:
> >
> > 1. Go back to defines.
> > 2. Don't pass a ptr to func, but pass an enum which indicates whether
> >     to use bpf_prog_run or __bpf_prog_run_save_cb. Seems like in this
> >     case the compiler shouldn't have any trouble unwrapping it?
> >
> > I'll prototype and send (2). If it won't work out we can always get back
> > to (1).
>
> Going back to defines is probably not necessary.
> Could you try moving bpf_prog_run_array_cg*() back to .h
> and use static __always_inline ?

Actually below was enough for gcc 8 and 10:
-static int
+static __always_inline int
 bpf_prog_run_array_cg_flags(const struct cgroup_bpf *cgrp,
                            enum cgroup_bpf_attach_type atype,
                            const void *ctx, bpf_prog_run_fn run_prog,
@@ -55,7 +55,7 @@ bpf_prog_run_array_cg_flags(const struct cgroup_bpf *cgrp,
        return run_ctx.retval;
 }

-static int
+static __always_inline int
 bpf_prog_run_array_cg(const struct cgroup_bpf *cgrp,

we can keep them in .c and generated code looks good.

I can apply it with the above change.
wdyt?



[Index of Archives]     [Linux Samsung SoC]     [Linux Rockchip SoC]     [Linux Actions SoC]     [Linux for Synopsys ARC Processors]     [Linux NFS]     [Linux NILFS]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]


  Powered by Linux