On Thu, Nov 10, 2022 at 2:55 AM Joel Fernandes (Google)
<joel@xxxxxxxxxxxxxxxxx> wrote:
>
> Currently ftrace only dumps the current trace buffer on OOPS. For
> debugging a production usecase, I'd like to dump instance traces as
> well, into the kernel logs. The reason is we cannot use the global trace
> buffer as it may be used for other purposes.
>
> This patch adds support for dumping the trace buffer instances along
> with the global trace buffer.
>
> The instance traces are dumped first, and then the global trace buffer.
>
> Cc: Ross Zwisler <zwisler@xxxxxxxxxx>
> Signed-off-by: Joel Fernandes (Google) <joel@xxxxxxxxxxxxxxxxx>
> ---
> kernel/trace/trace.c | 70 ++++++++++++++++++++++++--------------------
> 1 file changed, 39 insertions(+), 31 deletions(-)
>
> diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
> index 47a44b055a1d..15ddee31cc23 100644
> --- a/kernel/trace/trace.c
> +++ b/kernel/trace/trace.c
> @@ -9914,15 +9914,12 @@ trace_printk_seq(struct trace_seq *s)
> trace_seq_init(s);
> }
>
> -void trace_init_global_iter(struct trace_iterator *iter)
> +void trace_init_iter_with_tr(struct trace_iterator *iter, struct trace_array *tr)
> {
> - iter->tr = &global_trace;
> + iter->tr = tr;
> iter->trace = iter->tr->current_trace;
> iter->cpu_file = RING_BUFFER_ALL_CPUS;
> - iter->array_buffer = &global_trace.array_buffer;
> -
> - if (iter->trace && iter->trace->open)
> - iter->trace->open(iter);
I dropped this open() call by mistake. I did not show up as an issue
in my testing, so I missed it. I'll go fix that and send v2, sorry.
- Joel
> + iter->array_buffer = &iter->tr->array_buffer;
>
> /* Annotate start of buffers if we had overruns */
> if (ring_buffer_overruns(iter->array_buffer->buffer))
> @@ -9939,36 +9936,14 @@ void trace_init_global_iter(struct trace_iterator *iter)
> iter->fmt_size = STATIC_FMT_BUF_SIZE;
> }
>
> -void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
> +void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode oops_dump_mode)
> {
> - /* use static because iter can be a bit big for the stack */
> static struct trace_iterator iter;
> - static atomic_t dump_running;
> - struct trace_array *tr = &global_trace;
> unsigned int old_userobj;
> - unsigned long flags;
> int cnt = 0, cpu;
>
> - /* Only allow one dump user at a time. */
> - if (atomic_inc_return(&dump_running) != 1) {
> - atomic_dec(&dump_running);
> - return;
> - }
> -
> - /*
> - * Always turn off tracing when we dump.
> - * We don't need to show trace output of what happens
> - * between multiple crashes.
> - *
> - * If the user does a sysrq-z, then they can re-enable
> - * tracing with echo 1 > tracing_on.
> - */
> - tracing_off();
> -
> - local_irq_save(flags);
> -
> /* Simulate the iterator */
> - trace_init_global_iter(&iter);
> + trace_init_iter_with_tr(&iter, tr);
>
> for_each_tracing_cpu(cpu) {
> atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
> @@ -9993,7 +9968,10 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
> iter.cpu_file = RING_BUFFER_ALL_CPUS;
> }
>
> - printk(KERN_TRACE "Dumping ftrace buffer:\n");
> + if (tr == &global_trace)
> + printk(KERN_TRACE "Dumping ftrace buffer:\n");
> + else
> + printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
>
> /* Did function tracer already get disabled? */
> if (ftrace_is_dead()) {
> @@ -10041,6 +10019,36 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
> for_each_tracing_cpu(cpu) {
> atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
> }
> +}
> +
> +void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
> +{
> + /* use static because iter can be a bit big for the stack */
> + static atomic_t dump_running;
> + struct trace_array *tr;
> + unsigned long flags;
> +
> + /* Only allow one dump user at a time. */
> + if (atomic_inc_return(&dump_running) != 1) {
> + atomic_dec(&dump_running);
> + return;
> + }
> +
> + /*
> + * Always turn off tracing when we dump.
> + * We don't need to show trace output of what happens
> + * between multiple crashes.
> + *
> + * If the user does a sysrq-z, then they can re-enable
> + * tracing with echo 1 > tracing_on.
> + */
> + tracing_off();
> + local_irq_save(flags);
> +
> + list_for_each_entry(tr, &ftrace_trace_arrays, list) {
> + ftrace_dump_one(tr, oops_dump_mode);
> + }
> +
> atomic_dec(&dump_running);
> local_irq_restore(flags);
> }
> --
> 2.38.1.493.g58b659f92b-goog
>
|