On Tue, Jun 20, 2023 at 1:38 AM Jiri Olsa <jolsa@xxxxxxxxxx> wrote:
>
> Adding support for usdt_manager_attach_usdt to use uprobe_multi
> link to attach to usdt probes.
>
> The uprobe_multi support is detected before the usdt program is
> loaded and its expected_attach_type is set accordingly.
>
> If uprobe_multi support is detected the usdt_manager_attach_usdt
> gathers uprobes info and calls bpf_program__attach_uprobe_opts to
> create all needed uprobes.
>
> If uprobe_multi support is not detected the old behaviour stays.
>
> Signed-off-by: Jiri Olsa <jolsa@xxxxxxxxxx>
> ---
> tools/lib/bpf/libbpf.c | 12 ++++-
> tools/lib/bpf/usdt.c | 120 ++++++++++++++++++++++++++++++-----------
> 2 files changed, 99 insertions(+), 33 deletions(-)
>
> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
> index 3d570898459e..9c7a67c5cbe8 100644
> --- a/tools/lib/bpf/libbpf.c
> +++ b/tools/lib/bpf/libbpf.c
> @@ -363,6 +363,8 @@ enum sec_def_flags {
> SEC_SLEEPABLE = 8,
> /* BPF program support non-linear XDP buffer */
> SEC_XDP_FRAGS = 16,
> + /* Setup proper attach type for usdt probes. */
> + SEC_USDT = 32,
> };
>
> struct bpf_sec_def {
> @@ -6799,6 +6801,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog,
> if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
> opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
>
> + /* special check for usdt to use uprobe_multi link */
> + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_LINK))
> + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
this is quite ugly. I think KPROBE programs do not have enforcement
for expected_attach_type during BPF_PROG_LOAD, so we can set
BPF_TRACE_UPROBE_MULTI unconditionally, right?
> +
> if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
> int btf_obj_fd = 0, btf_type_id = 0, err;
> const char *attach_name;
> @@ -6867,7 +6873,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
> if (!insns || !insns_cnt)
> return -EINVAL;
>
> - load_attr.expected_attach_type = prog->expected_attach_type;
> if (kernel_supports(obj, FEAT_PROG_NAME))
> prog_name = prog->name;
> load_attr.attach_prog_fd = prog->attach_prog_fd;
> @@ -6903,6 +6908,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
> insns_cnt = prog->insns_cnt;
> }
>
> + /* allow prog_prepare_load_fn to change expected_attach_type */
> + load_attr.expected_attach_type = prog->expected_attach_type;
> +
> if (obj->gen_loader) {
> bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
> license, insns, insns_cnt, &load_attr,
> @@ -8703,7 +8711,7 @@ static const struct bpf_sec_def section_defs[] = {
> SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
> SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
> SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
> - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
> + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
btw, given you are touching USDT stuff, can you please also add
sleepable USDT (usdt.s+) ?
> SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE),
> SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE),
> SEC_DEF("action", SCHED_ACT, 0, SEC_NONE),
> diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
> index f1a141555f08..33f0a2b4cc1c 100644
> --- a/tools/lib/bpf/usdt.c
> +++ b/tools/lib/bpf/usdt.c
> @@ -808,6 +808,16 @@ struct bpf_link_usdt {
> long abs_ip;
> struct bpf_link *link;
> } *uprobes;
> +
> + bool has_uprobe_multi;
> +
> + struct {
> + char *path;
> + unsigned long *offsets;
> + unsigned long *ref_ctr_offsets;
> + __u64 *cookies;
you shouldn't need to persist this, this can be allocated and freed
inside usdt_manager_attach_usdt(), you only need the link pointer
> + struct bpf_link *link;
> + } uprobe_multi;
> };
>
> static int bpf_link_usdt_detach(struct bpf_link *link)
> @@ -816,19 +826,23 @@ static int bpf_link_usdt_detach(struct bpf_link *link)
> struct usdt_manager *man = usdt_link->usdt_man;
> int i;
>
> - for (i = 0; i < usdt_link->uprobe_cnt; i++) {
> - /* detach underlying uprobe link */
> - bpf_link__destroy(usdt_link->uprobes[i].link);
> - /* there is no need to update specs map because it will be
> - * unconditionally overwritten on subsequent USDT attaches,
> - * but if BPF cookies are not used we need to remove entry
> - * from ip_to_spec_id map, otherwise we'll run into false
> - * conflicting IP errors
> - */
> - if (!man->has_bpf_cookie) {
> - /* not much we can do about errors here */
> - (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map),
> - &usdt_link->uprobes[i].abs_ip);
> + if (usdt_link->has_uprobe_multi) {
> + bpf_link__destroy(usdt_link->uprobe_multi.link);
> + } else {
> + for (i = 0; i < usdt_link->uprobe_cnt; i++) {
> + /* detach underlying uprobe link */
> + bpf_link__destroy(usdt_link->uprobes[i].link);
> + /* there is no need to update specs map because it will be
> + * unconditionally overwritten on subsequent USDT attaches,
> + * but if BPF cookies are not used we need to remove entry
> + * from ip_to_spec_id map, otherwise we'll run into false
> + * conflicting IP errors
> + */
> + if (!man->has_bpf_cookie) {
> + /* not much we can do about errors here */
> + (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map),
> + &usdt_link->uprobes[i].abs_ip);
> + }
> }
you can avoid shifting all this by keeping uprobe_cnt to zero
bpf_link__destory(usdt_link->uprobe_multi.link) will work fine for NULL
so just do both clean ups sequentially, knowing that only one of them
will actually do anything
> }
>
> @@ -868,9 +882,15 @@ static void bpf_link_usdt_dealloc(struct bpf_link *link)
> {
> struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link);
>
> - free(usdt_link->spec_ids);
> - free(usdt_link->uprobes);
> - free(usdt_link);
> + if (usdt_link->has_uprobe_multi) {
> + free(usdt_link->uprobe_multi.offsets);
> + free(usdt_link->uprobe_multi.ref_ctr_offsets);
> + free(usdt_link->uprobe_multi.cookies);
> + } else {
> + free(usdt_link->spec_ids);
> + free(usdt_link->uprobes);
> + free(usdt_link);
> + }
similar to the above, just do *all* the clean up unconditionally and
rely on free() handling NULLs just fine
> }
>
> static size_t specs_hash_fn(long key, void *ctx)
> @@ -943,11 +963,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
> const char *usdt_provider, const char *usdt_name,
> __u64 usdt_cookie)
> {
> + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi);
> int i, fd, err, spec_map_fd, ip_map_fd;
> LIBBPF_OPTS(bpf_uprobe_opts, opts);
> struct hashmap *specs_hash = NULL;
> struct bpf_link_usdt *link = NULL;
> struct usdt_target *targets = NULL;
> + struct bpf_link *uprobe_link;
> size_t target_cnt;
> Elf *elf;
>
> @@ -1003,16 +1025,29 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
> link->usdt_man = man;
> link->link.detach = &bpf_link_usdt_detach;
> link->link.dealloc = &bpf_link_usdt_dealloc;
> + link->has_uprobe_multi = bpf_program__expected_attach_type(prog) == BPF_TRACE_UPROBE_MULTI;
just use kernel_supports(), it's cleaner (and result is cached, so
it's not less efficient)
>
> - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
> - if (!link->uprobes) {
> - err = -ENOMEM;
> - goto err_out;
> + if (link->has_uprobe_multi) {
> + link->uprobe_multi.offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.offsets));
> + link->uprobe_multi.ref_ctr_offsets = calloc(target_cnt, sizeof(*link->uprobe_multi.ref_ctr_offsets));
> + link->uprobe_multi.cookies = calloc(target_cnt, sizeof(*link->uprobe_multi.cookies));
> +
> + if (!link->uprobe_multi.offsets ||
> + !link->uprobe_multi.ref_ctr_offsets ||
> + !link->uprobe_multi.cookies) {
> + err = -ENOMEM;
> + goto err_out;
> + }
> + } else {
> + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
> + if (!link->uprobes) {
> + err = -ENOMEM;
> + goto err_out;
> + }
> }
>
[...]
