Hello,
Any feedback on this?
On Thu, Jan 5, 2023 at 12:32 PM Namhyung Kim <namhyung@xxxxxxxxxx> wrote:
>
> When there're many lock contentions in the system, people sometimes
> want to know who caused the contention, IOW who's the owner of the
> locks.
>
> The -o/--lock-owner option tries to follow the lock owners for the
> contended mutexes and rwsems from BPF, and then attributes the
> contention time to the owner instead of the waiter. It's a best
> effort approach to get the owner info at the time of the contention
> and doesn't guarantee to have the precise tracking of owners if it's
> changing over time.
>
> Currently it only handles mutex and rwsem that have owner field in
> their struct and it basically points to a task_struct that owns the
> lock at the moment.
>
> Technically its type is atomic_long_t and it comes with some LSB bits
> used for other meaninigs. So it needs to clear them when casting it
> to a pointer to task_struct.
>
> Also the atomic_long_t is a typedef of the atomic 32 or 64 bit types
> depending on arch which is a wrapper struct for the counter value.
> I'm not aware of proper ways to access those kernel atomic types from
> BPF so I just read the internal counter value directly. Please let me
> know if there's a better way.
>
> When -o/--lock-owner option is used, it goes to the task aggregation
> mode like -t/--threads option does. However it cannot get the owner
> for other lock types like spinlock and sometimes even for mutex.
>
> $ sudo ./perf lock con -abo -- ./perf bench sched pipe
> # Running 'sched/pipe' benchmark:
> # Executed 1000000 pipe operations between two processes
>
> Total time: 4.766 [sec]
>
> 4.766540 usecs/op
> 209795 ops/sec
> contended total wait max wait avg wait pid owner
>
> 403 565.32 us 26.81 us 1.40 us -1 Unknown
> 4 27.99 us 8.57 us 7.00 us 1583145 sched-pipe
> 1 8.25 us 8.25 us 8.25 us 1583144 sched-pipe
> 1 2.03 us 2.03 us 2.03 us 5068 chrome
>
> As you can see, the owner is unknown for the most cases. But if we
> filter only for the mutex locks, it'd more likely get the onwers.
>
> $ sudo ./perf lock con -abo -Y mutex -- ./perf bench sched pipe
> # Running 'sched/pipe' benchmark:
> # Executed 1000000 pipe operations between two processes
>
> Total time: 4.910 [sec]
>
> 4.910435 usecs/op
> 203647 ops/sec
> contended total wait max wait avg wait pid owner
>
> 2 15.50 us 8.29 us 7.75 us 1582852 sched-pipe
> 7 7.20 us 2.47 us 1.03 us -1 Unknown
> 1 6.74 us 6.74 us 6.74 us 1582851 sched-pipe
>
> Signed-off-by: Namhyung Kim <namhyung@xxxxxxxxxx>
> ---
> tools/perf/Documentation/perf-lock.txt | 5 ++
> tools/perf/builtin-lock.c | 49 ++++++++++++---
> tools/perf/util/bpf_lock_contention.c | 1 +
> .../perf/util/bpf_skel/lock_contention.bpf.c | 60 +++++++++++++++++--
> tools/perf/util/lock-contention.h | 1 +
> 5 files changed, 102 insertions(+), 14 deletions(-)
>
> diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt
> index 0f9f720e599d..a41c8acc7002 100644
> --- a/tools/perf/Documentation/perf-lock.txt
> +++ b/tools/perf/Documentation/perf-lock.txt
> @@ -172,6 +172,11 @@ CONTENTION OPTIONS
> --lock-addr::
> Show lock contention stat by address
>
> +-o::
> +--lock-owner::
> + Show lock contention stat by owners. Implies --threads and
> + requires --use-bpf.
> +
> -Y::
> --type-filter=<value>::
> Show lock contention only for given lock types (comma separated list).
> diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c
> index 718b82bfcdff..5a3ed5a2bd3d 100644
> --- a/tools/perf/builtin-lock.c
> +++ b/tools/perf/builtin-lock.c
> @@ -58,6 +58,7 @@ static struct rb_root thread_stats;
> static bool combine_locks;
> static bool show_thread_stats;
> static bool show_lock_addrs;
> +static bool show_lock_owner;
> static bool use_bpf;
> static unsigned long bpf_map_entries = 10240;
> static int max_stack_depth = CONTENTION_STACK_DEPTH;
> @@ -1567,7 +1568,8 @@ static void print_contention_result(struct lock_contention *con)
>
> switch (aggr_mode) {
> case LOCK_AGGR_TASK:
> - pr_info(" %10s %s\n\n", "pid", "comm");
> + pr_info(" %10s %s\n\n", "pid",
> + show_lock_owner ? "owner" : "comm");
> break;
> case LOCK_AGGR_CALLER:
> pr_info(" %10s %s\n\n", "type", "caller");
> @@ -1607,7 +1609,8 @@ static void print_contention_result(struct lock_contention *con)
> case LOCK_AGGR_TASK:
> pid = st->addr;
> t = perf_session__findnew(session, pid);
> - pr_info(" %10d %s\n", pid, thread__comm_str(t));
> + pr_info(" %10d %s\n",
> + pid, pid == -1 ? "Unknown" : thread__comm_str(t));
> break;
> case LOCK_AGGR_ADDR:
> pr_info(" %016llx %s\n", (unsigned long long)st->addr,
> @@ -1718,6 +1721,37 @@ static void sighandler(int sig __maybe_unused)
> {
> }
>
> +static int check_lock_contention_options(const struct option *options,
> + const char * const *usage)
> +
> +{
> + if (show_thread_stats && show_lock_addrs) {
> + pr_err("Cannot use thread and addr mode together\n");
> + parse_options_usage(usage, options, "threads", 0);
> + parse_options_usage(NULL, options, "lock-addr", 0);
> + return -1;
> + }
> +
> + if (show_lock_owner && !use_bpf) {
> + pr_err("Lock owners are available only with BPF\n");
> + parse_options_usage(usage, options, "lock-owner", 0);
> + parse_options_usage(NULL, options, "use-bpf", 0);
> + return -1;
> + }
> +
> + if (show_lock_owner && show_lock_addrs) {
> + pr_err("Cannot use owner and addr mode together\n");
> + parse_options_usage(usage, options, "lock-owner", 0);
> + parse_options_usage(NULL, options, "lock-addr", 0);
> + return -1;
> + }
> +
> + if (show_lock_owner)
> + show_thread_stats = true;
> +
> + return 0;
> +}
> +
> static int __cmd_contention(int argc, const char **argv)
> {
> int err = -EINVAL;
> @@ -1742,6 +1776,7 @@ static int __cmd_contention(int argc, const char **argv)
> .max_stack = max_stack_depth,
> .stack_skip = stack_skip,
> .filters = &filters,
> + .owner = show_lock_owner,
> };
>
> session = perf_session__new(use_bpf ? NULL : &data, &eops);
> @@ -2188,6 +2223,7 @@ int cmd_lock(int argc, const char **argv)
> "Filter specific type of locks", parse_lock_type),
> OPT_CALLBACK('L', "lock-filter", NULL, "ADDRS/NAMES",
> "Filter specific address/symbol of locks", parse_lock_addr),
> + OPT_BOOLEAN('o', "lock-owner", &show_lock_owner, "show lock owners instead of waiters"),
> OPT_PARENT(lock_options)
> };
>
> @@ -2258,14 +2294,9 @@ int cmd_lock(int argc, const char **argv)
> contention_usage, 0);
> }
>
> - if (show_thread_stats && show_lock_addrs) {
> - pr_err("Cannot use thread and addr mode together\n");
> - parse_options_usage(contention_usage, contention_options,
> - "threads", 0);
> - parse_options_usage(NULL, contention_options,
> - "lock-addr", 0);
> + if (check_lock_contention_options(contention_options,
> + contention_usage) < 0)
> return -1;
> - }
>
> rc = __cmd_contention(argc, argv);
> } else {
> diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c
> index 0236334fd69b..709d418be873 100644
> --- a/tools/perf/util/bpf_lock_contention.c
> +++ b/tools/perf/util/bpf_lock_contention.c
> @@ -146,6 +146,7 @@ int lock_contention_prepare(struct lock_contention *con)
> /* these don't work well if in the rodata section */
> skel->bss->stack_skip = con->stack_skip;
> skel->bss->aggr_mode = con->aggr_mode;
> + skel->bss->lock_owner = con->owner;
>
> lock_contention_bpf__attach(skel);
> return 0;
> diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c
> index ad0ca5d50557..a035a267b08e 100644
> --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c
> +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c
> @@ -10,6 +10,14 @@
> /* default buffer size */
> #define MAX_ENTRIES 10240
>
> +/* lock contention flags from include/trace/events/lock.h */
> +#define LCB_F_SPIN (1U << 0)
> +#define LCB_F_READ (1U << 1)
> +#define LCB_F_WRITE (1U << 2)
> +#define LCB_F_RT (1U << 3)
> +#define LCB_F_PERCPU (1U << 4)
> +#define LCB_F_MUTEX (1U << 5)
> +
> struct tstamp_data {
> __u64 timestamp;
> __u64 lock;
> @@ -83,6 +91,7 @@ int has_task;
> int has_type;
> int has_addr;
> int stack_skip;
> +int lock_owner;
>
> /* determine the key of lock stat */
> int aggr_mode;
> @@ -131,17 +140,24 @@ static inline int can_record(u64 *ctx)
> return 1;
> }
>
> -static inline void update_task_data(__u32 pid)
> +static inline int update_task_data(struct task_struct *task)
> {
> struct contention_task_data *p;
> + int pid, err;
> +
> + err = bpf_core_read(&pid, sizeof(pid), &task->pid);
> + if (err)
> + return -1;
>
> p = bpf_map_lookup_elem(&task_data, &pid);
> if (p == NULL) {
> - struct contention_task_data data;
> + struct contention_task_data data = {};
>
> - bpf_get_current_comm(data.comm, sizeof(data.comm));
> + BPF_CORE_READ_STR_INTO(&data.comm, task, comm);
> bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST);
> }
> +
> + return 0;
> }
>
> SEC("tp_btf/contention_begin")
> @@ -178,6 +194,38 @@ int contention_begin(u64 *ctx)
> BPF_F_FAST_STACK_CMP | stack_skip);
> if (pelem->stack_id < 0)
> lost++;
> + } else if (aggr_mode == LOCK_AGGR_TASK) {
> + struct task_struct *task;
> +
> + if (lock_owner) {
> + if (pelem->flags & LCB_F_MUTEX) {
> + struct mutex *lock = (void *)pelem->lock;
> + unsigned long owner = BPF_CORE_READ(lock, owner.counter);
> +
> + task = (void *)(owner & ~7UL);
> + } else if (pelem->flags == LCB_F_READ || pelem->flags == LCB_F_WRITE) {
> + struct rw_semaphore *lock = (void *)pelem->lock;
> + unsigned long owner = BPF_CORE_READ(lock, owner.counter);
> +
> + task = (void *)(owner & ~7UL);
> + } else {
> + task = NULL;
> + }
> +
> + /* The flags is not used anymore. Pass the owner pid. */
> + if (task)
> + pelem->flags = BPF_CORE_READ(task, pid);
> + else
> + pelem->flags = -1U;
> +
> + } else {
> + task = bpf_get_current_task_btf();
> + }
> +
> + if (task) {
> + if (update_task_data(task) < 0 && lock_owner)
> + pelem->flags = -1U;
> + }
> }
>
> return 0;
> @@ -207,8 +255,10 @@ int contention_end(u64 *ctx)
> key.aggr_key = pelem->stack_id;
> break;
> case LOCK_AGGR_TASK:
> - key.aggr_key = pid;
> - update_task_data(pid);
> + if (lock_owner)
> + key.aggr_key = pelem->flags;
> + else
> + key.aggr_key = pid;
> break;
> case LOCK_AGGR_ADDR:
> key.aggr_key = pelem->lock;
> diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h
> index b99e83fccf5c..862b0617698a 100644
> --- a/tools/perf/util/lock-contention.h
> +++ b/tools/perf/util/lock-contention.h
> @@ -128,6 +128,7 @@ struct lock_contention {
> int max_stack;
> int stack_skip;
> int aggr_mode;
> + int owner;
> };
>
> #ifdef HAVE_BPF_SKEL
> --
> 2.39.0.314.g84b9a713c41-goog
>
