On Fri, Jun 30, 2023 at 1:37 AM Jiri Olsa <jolsa@xxxxxxxxxx> wrote:
>
> Adding support for usdt_manager_attach_usdt to use uprobe_multi
> link to attach to usdt probes.
>
> The uprobe_multi support is detected before the usdt program is
> loaded and its expected_attach_type is set accordingly.
>
> If uprobe_multi support is detected the usdt_manager_attach_usdt
> gathers uprobes info and calls bpf_program__attach_uprobe to
> create all needed uprobes.
>
> If uprobe_multi support is not detected the old behaviour stays.
>
> Also adding usdt.s program section for sleepable usdt probes.
>
> Signed-off-by: Jiri Olsa <jolsa@xxxxxxxxxx>
> ---
> tools/lib/bpf/libbpf.c | 13 +++++--
> tools/lib/bpf/usdt.c | 78 ++++++++++++++++++++++++++++++++++--------
> 2 files changed, 75 insertions(+), 16 deletions(-)
>
> diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
> index 4f61f9dc1748..e234c2e860f9 100644
> --- a/tools/lib/bpf/libbpf.c
> +++ b/tools/lib/bpf/libbpf.c
> @@ -365,6 +365,8 @@ enum sec_def_flags {
> SEC_SLEEPABLE = 8,
> /* BPF program support non-linear XDP buffer */
> SEC_XDP_FRAGS = 16,
> + /* Setup proper attach type for usdt probes. */
> + SEC_USDT = 32,
> };
>
> struct bpf_sec_def {
> @@ -6807,6 +6809,10 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog,
> if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
> opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
>
> + /* special check for usdt to use uprobe_multi link */
> + if ((def & SEC_USDT) && kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK))
please pass prog->obj to kernel_supports(), it will be especially
important later with BPF token stuff
> + prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
> +
> if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
> int btf_obj_fd = 0, btf_type_id = 0, err;
> const char *attach_name;
> @@ -6875,7 +6881,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
> if (!insns || !insns_cnt)
> return -EINVAL;
>
> - load_attr.expected_attach_type = prog->expected_attach_type;
> if (kernel_supports(obj, FEAT_PROG_NAME))
> prog_name = prog->name;
> load_attr.attach_prog_fd = prog->attach_prog_fd;
> @@ -6911,6 +6916,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog
> insns_cnt = prog->insns_cnt;
> }
>
> + /* allow prog_prepare_load_fn to change expected_attach_type */
> + load_attr.expected_attach_type = prog->expected_attach_type;
> +
> if (obj->gen_loader) {
> bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
> license, insns, insns_cnt, &load_attr,
> @@ -8711,7 +8719,8 @@ static const struct bpf_sec_def section_defs[] = {
> SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
> SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
> SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
> - SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
> + SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
> + SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT|SEC_SLEEPABLE, attach_usdt),
spaces around |
> SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE),
> SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE),
> SEC_DEF("action", SCHED_ACT, 0, SEC_NONE),
> diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
> index 9fa883ebc0bd..6ff66a8eaf85 100644
> --- a/tools/lib/bpf/usdt.c
> +++ b/tools/lib/bpf/usdt.c
> @@ -809,6 +809,8 @@ struct bpf_link_usdt {
> long abs_ip;
> struct bpf_link *link;
> } *uprobes;
> +
> + struct bpf_link *multi_link;
> };
>
> static int bpf_link_usdt_detach(struct bpf_link *link)
> @@ -817,6 +819,9 @@ static int bpf_link_usdt_detach(struct bpf_link *link)
> struct usdt_manager *man = usdt_link->usdt_man;
> int i;
>
> + /* When having multi_link, uprobe_cnt is 0 */
misplaced comment, move down to for() loop?
> + bpf_link__destroy(usdt_link->multi_link);
> +
> for (i = 0; i < usdt_link->uprobe_cnt; i++) {
> /* detach underlying uprobe link */
> bpf_link__destroy(usdt_link->uprobes[i].link);
> @@ -944,11 +949,13 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
> const char *usdt_provider, const char *usdt_name,
> __u64 usdt_cookie)
> {
> + unsigned long *offsets = NULL, *ref_ctr_offsets = NULL;
> int i, err, spec_map_fd, ip_map_fd;
> LIBBPF_OPTS(bpf_uprobe_opts, opts);
> struct hashmap *specs_hash = NULL;
> struct bpf_link_usdt *link = NULL;
> struct usdt_target *targets = NULL;
> + __u64 *cookies = NULL;
> struct elf_fd elf_fd;
> size_t target_cnt;
>
> @@ -995,10 +1002,21 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
> link->link.detach = &bpf_link_usdt_detach;
> link->link.dealloc = &bpf_link_usdt_dealloc;
>
> - link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
> - if (!link->uprobes) {
> - err = -ENOMEM;
> - goto err_out;
> + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) {
see how we feature-detect has_sema_refcnt and has_bpf_cookie, let's do
the same with UPROBE_MULTI_LINK, detect once, remember, consistently
use it (it also matters later for BPF token)
> + offsets = calloc(target_cnt, sizeof(*offsets));
> + cookies = calloc(target_cnt, sizeof(*cookies));
> + ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets));
> +
> + if (!offsets || !ref_ctr_offsets || !cookies) {
> + err = -ENOMEM;
> + goto err_out;
> + }
> + } else {
> + link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
> + if (!link->uprobes) {
> + err = -ENOMEM;
> + goto err_out;
> + }
> }
>
> for (i = 0; i < target_cnt; i++) {
> @@ -1039,20 +1057,48 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
> goto err_out;
> }
>
> - opts.ref_ctr_offset = target->sema_off;
> - opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
> - uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
> - target->rel_ip, &opts);
> - err = libbpf_get_error(uprobe_link);
> + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) {
> + offsets[i] = target->rel_ip;
> + ref_ctr_offsets[i] = target->sema_off;
> + cookies[i] = spec_id;
> + } else {
> + opts.ref_ctr_offset = target->sema_off;
> + opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
> + uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
> + target->rel_ip, &opts);
> + err = libbpf_get_error(uprobe_link);
> + if (err) {
> + pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
> + i, usdt_provider, usdt_name, path, err);
> + goto err_out;
> + }
> +
> + link->uprobes[i].link = uprobe_link;
> + link->uprobes[i].abs_ip = target->abs_ip;
> + link->uprobe_cnt++;
> + }
> + }
> +
> + if (kernel_supports(NULL, FEAT_UPROBE_MULTI_LINK)) {
same as above, we should feature-detect once per usdt_manager while we
have associated bpf_object
> + LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi,
> + .cnt = target_cnt,
> + .offsets = offsets,
> + .ref_ctr_offsets = ref_ctr_offsets,
> + .cookies = cookies,
> + );
> +
> + link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path,
> + NULL, &opts_multi);
> + err = libbpf_get_error(link->multi_link);
let's not use libbpf_get_error() in new code, there is no need, just
`err = -errno` and `if (!link->multi_link)`
> if (err) {
> - pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",
> - i, usdt_provider, usdt_name, path, err);
> + pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",
> + usdt_provider, usdt_name, path, err);
> goto err_out;
> }
>
> - link->uprobes[i].link = uprobe_link;
> - link->uprobes[i].abs_ip = target->abs_ip;
> - link->uprobe_cnt++;
> + free(offsets);
> + free(ref_ctr_offsets);
> + free(cookies);
> }
>
> free(targets);
> @@ -1061,6 +1107,10 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
> return &link->link;
>
> err_out:
> + free(offsets);
> + free(ref_ctr_offsets);
> + free(cookies);
> +
> if (link)
> bpf_link__destroy(&link->link);
> free(targets);
> --
> 2.41.0
>
