On Tue, 21 Mar 2023 16:15:53 +0100 Uladzislau Rezki <urezki@xxxxxxxxx> wrote: > Collect traces as much as you want: XQ-DQ54:/sys/kernel/tracing # echo 1 > tracing_on; sleep 10; echo 0 > tracing_on > Next problem is how to parse it. Of course you will not be able to parse > megabytes of traces. For that purpose i use a special C trace parser. > If you need an example please let me know i can show here. Not sure if you are familiar with trace-cmd, but the above could have been: # trace-cmd record -e rcu sleep 10 and then you get the trace.dat file, which reports as: # trace-cmd report If you need special parsing, there's a libtracecmd library that lets you do all that! https://www.trace-cmd.org/Documentation/libtracecmd/ And for parsing events: https://www.trace-cmd.org/Documentation/libtraceevent/ Basically have: struct my_info { /* store state info here */ }; int main(...) { struct tracecmd_input *handle; struct my_info info; char *file = argv[1]; handle = tracecmd_open(file); tracecmd_follow_event(handle, "rcu", "rcu_batch_start", batch_start, &info); tracecmd_follow_event(handle, "rcu", "rcu_batch_end", batch_end, &info); tracecmd_follow_event(handle, "rcu", "rcu_invoke_callback", invoke_callback, &info); tracecmd_iterate_events(handle, NULL, 0, NULL, NULL); tracecmd_close(handle); } Where it will iterate the "trace.dat" file passed it, and every time it hits an event registered by follow_event it will call that function: static int batch_start(struct tracecmd_input *handle, struct tep_event *event, struct tep_record *record, int cpu, void *data) { struct my_info *info = data; info->start_timestamp = record->ts; return 0; } static int batch_end(struct tracecmd_input *handle, struct tep_event *event, struct tep_record *record, int cpu, void *data) { struct my_info *info = data; printf("time = %lld -> %lld\n", info->start_timestapm, record->ts); return 0; } static int invoke_callback(struct tracecmd_input *handle, struct tep_event *event, struct tep_record *record, int cpu, void *data) { struct my_info *info = data; struct tep_handle *tep = tracecmd_get_tep(handle); static struct tep_format_field *ip_field; unsigned long long ip; const char *func; int pid; if (!ip_field) ip_field = tep_find_field(event, "func"); tep_read_number_field(ip_field, record->data, &ip); func = tep_find_function(tep, ip); pid = tep_data_pid(tep, record); if (func) printf("Processing function %s for pid %d\n", func, pid); else printf("Processing address 0x%llx for pid %d\n", ip, pid); return 0; } And much more ;-) Oh, and if you just want to read the live trace without recording, you could always use libtracefs: https://www.trace-cmd.org/Documentation/libtracefs/ And instead of using tracecmd_follow_event() with tracecmd_iterate_events(), you can use: const char *systems = { "rcu" }; tep = tracefs_local_events_systems(NULL, systems); tracefs_follow_event(tep, NULL, "rcu", "rcu_invoke_callback", invoke_callback, &info); and iterate the live events with: tracefs_iterate_raw_events(tep, NULL, NULL, 0, NULL, NULL); With the callback for this being (very similar): static int invoke_callback(struct tep_event *event, struct tep_record *record, int cpu, void *data) { struct my_info *info = data; struct tep_handle *tep = event->tep; static struct tep_format_field *ip_field; unsigned long long ip; const char *func; int pid; if (!ip_field) ip_field = tep_find_field(event, "func"); tep_read_number_field(ip_field, record->data, &ip); func = tep_find_function(tep, ip); pid = tep_data_pid(tep, record); if (func) printf("Processing function %s for pid %d\n", func, pid); else printf("Processing address 0x%llx for pid %d\n", ip, pid); return 0; } -- Steve