Re: [PATCH] tracing: Dump instance traces into oops with ftrace_dump_on_oops

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Thu, Nov 10, 2022 at 2:55 AM Joel Fernandes (Google)
<joel@xxxxxxxxxxxxxxxxx> wrote:
>
> Currently ftrace only dumps the current trace buffer on OOPS. For
> debugging a production usecase, I'd like to dump instance traces as
> well, into the kernel logs. The reason is we cannot use the global trace
> buffer as it may be used for other purposes.
>
> This patch adds support for dumping the trace buffer instances along
> with the global trace buffer.
>
> The instance traces are dumped first, and then the global trace buffer.
>
> Cc: Ross Zwisler <zwisler@xxxxxxxxxx>
> Signed-off-by: Joel Fernandes (Google) <joel@xxxxxxxxxxxxxxxxx>
> ---
>  kernel/trace/trace.c | 70 ++++++++++++++++++++++++--------------------
>  1 file changed, 39 insertions(+), 31 deletions(-)
>
> diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
> index 47a44b055a1d..15ddee31cc23 100644
> --- a/kernel/trace/trace.c
> +++ b/kernel/trace/trace.c
> @@ -9914,15 +9914,12 @@ trace_printk_seq(struct trace_seq *s)
>         trace_seq_init(s);
>  }
>
> -void trace_init_global_iter(struct trace_iterator *iter)
> +void trace_init_iter_with_tr(struct trace_iterator *iter, struct trace_array *tr)
>  {
> -       iter->tr = &global_trace;
> +       iter->tr = tr;
>         iter->trace = iter->tr->current_trace;
>         iter->cpu_file = RING_BUFFER_ALL_CPUS;
> -       iter->array_buffer = &global_trace.array_buffer;
> -
> -       if (iter->trace && iter->trace->open)
> -               iter->trace->open(iter);

I dropped this open() call by mistake. I did not show up as an issue
in my testing, so I missed it. I'll go fix that and send v2, sorry.

 - Joel


> +       iter->array_buffer = &iter->tr->array_buffer;
>
>         /* Annotate start of buffers if we had overruns */
>         if (ring_buffer_overruns(iter->array_buffer->buffer))
> @@ -9939,36 +9936,14 @@ void trace_init_global_iter(struct trace_iterator *iter)
>         iter->fmt_size = STATIC_FMT_BUF_SIZE;
>  }
>
> -void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
> +void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode oops_dump_mode)
>  {
> -       /* use static because iter can be a bit big for the stack */
>         static struct trace_iterator iter;
> -       static atomic_t dump_running;
> -       struct trace_array *tr = &global_trace;
>         unsigned int old_userobj;
> -       unsigned long flags;
>         int cnt = 0, cpu;
>
> -       /* Only allow one dump user at a time. */
> -       if (atomic_inc_return(&dump_running) != 1) {
> -               atomic_dec(&dump_running);
> -               return;
> -       }
> -
> -       /*
> -        * Always turn off tracing when we dump.
> -        * We don't need to show trace output of what happens
> -        * between multiple crashes.
> -        *
> -        * If the user does a sysrq-z, then they can re-enable
> -        * tracing with echo 1 > tracing_on.
> -        */
> -       tracing_off();
> -
> -       local_irq_save(flags);
> -
>         /* Simulate the iterator */
> -       trace_init_global_iter(&iter);
> +       trace_init_iter_with_tr(&iter, tr);
>
>         for_each_tracing_cpu(cpu) {
>                 atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
> @@ -9993,7 +9968,10 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
>                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
>         }
>
> -       printk(KERN_TRACE "Dumping ftrace buffer:\n");
> +       if (tr == &global_trace)
> +               printk(KERN_TRACE "Dumping ftrace buffer:\n");
> +       else
> +               printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
>
>         /* Did function tracer already get disabled? */
>         if (ftrace_is_dead()) {
> @@ -10041,6 +10019,36 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
>         for_each_tracing_cpu(cpu) {
>                 atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
>         }
> +}
> +
> +void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
> +{
> +       /* use static because iter can be a bit big for the stack */
> +       static atomic_t dump_running;
> +       struct trace_array *tr;
> +       unsigned long flags;
> +
> +       /* Only allow one dump user at a time. */
> +       if (atomic_inc_return(&dump_running) != 1) {
> +               atomic_dec(&dump_running);
> +               return;
> +       }
> +
> +       /*
> +        * Always turn off tracing when we dump.
> +        * We don't need to show trace output of what happens
> +        * between multiple crashes.
> +        *
> +        * If the user does a sysrq-z, then they can re-enable
> +        * tracing with echo 1 > tracing_on.
> +        */
> +       tracing_off();
> +       local_irq_save(flags);
> +
> +       list_for_each_entry(tr, &ftrace_trace_arrays, list) {
> +               ftrace_dump_one(tr, oops_dump_mode);
> +       }
> +
>         atomic_dec(&dump_running);
>         local_irq_restore(flags);
>  }
> --
> 2.38.1.493.g58b659f92b-goog
>



[Index of Archives]     [Linux USB Development]     [Linux USB Development]     [Linux Audio Users]     [Yosemite Hiking]     [Linux Kernel]     [Linux SCSI]

  Powered by Linux