Hello, I'd like to introduce a small patch for the crash tool (I described an idea behind it https://www.redhat.com/archives/crash-utility/2013-March/msg00164.html). * crash version 6.1.6 * arch x86_64 You can find it here https://github.com/hziSot/crash-stack-parser So, here are some results from a real system: =======> CPU #0 bin> bt PID: 0 TASK: ffffffff81a8d020 CPU: 0 COMMAND: "swapper" #0 [ffff880045603920] machine_kexec at ffffffff8103284b #1 [ffff880045603980] crash_kexec at ffffffff810ba972 #2 [ffff880045603a50] oops_end at ffffffff81501860 #3 [ffff880045603a80] no_context at ffffffff81043bfb #4 [ffff880045603ad0] __bad_area_nosemaphore at ffffffff81043e85 #5 [ffff880045603b20] bad_area_nosemaphore at ffffffff81043f53 #6 [ffff880045603b30] __do_page_fault at ffffffff810446b1 #7 [ffff880045603c50] do_page_fault at ffffffff8150383e #8 [ffff880045603c80] page_fault at ffffffff81500bf5 #9 [ffff880045603d60] activate_task at ffffffff81053953 #10 [ffff880045603d70] try_to_wake_up at ffffffff810600c0 #11 [ffff880045603de0] default_wake_function at ffffffff810602c2 #12 [ffff880045603df0] __wake_up_common at ffffffff8104e369 #13 [ffff880045603e40] complete at ffffffff81053347 #14 [ffff880045603e70] qla24xx_msix_default at ffffffffa00a8f74 [qla2xxx] #15 [ffff880045603ed0] handle_IRQ_event at ffffffff810dbb10 #16 [ffff880045603f20] handle_edge_irq at ffffffff810de29e #17 [ffff880045603f60] handle_irq at ffffffff8100df09 #18 [ffff880045603f80] do_IRQ at ffffffff815060bc --- <IRQ stack> --- #19 [ffffffff81a01da8] ret_from_intr at ffffffff8100ba53 [exception RIP: intel_idle+222] RIP: ffffffff812cdc0e RSP: ffffffff81a01e58 RFLAGS: 00000206 RAX: 0000000000000000 RBX: ffffffff81a01ec8 RCX: 0000000000000000 RDX: 000000000000116c RSI: 0000000000000000 RDI: 0000000000441165 RBP: ffffffff8100ba4e R8: 0000000000000005 R9: 000000000000006d R10: 000000c26ede6d76 R11: 0000000000000000 R12: ffff8800456115a0 R13: 0000000000000000 R14: 000000c12ce6a080 R15: ffff8800456116a0 ORIG_RAX: ffffffffffffff9d CS: 0010 SS: 0018 #20 [ffffffff81a01ed0] cpuidle_idle_call at ffffffff81407997 #21 [ffffffff81a01ef0] cpu_idle at ffffffff81009e06 bin> bt -H Backtrace: # 0: [RSP: 0xffff880045603980, RIP: 0xffffffff8103284b] machine_kexec (struct kimage * arg = 0xffff88082659b400) # 1: [RSP: 0xffff880045603a50, RIP: 0xffffffff810ba972] crash_kexec (struct pt_regs * arg = 0xffff880045603c88) # 2: [RSP: 0xffff880045603a80, RIP: 0xffffffff81501860] oops_end (long unsigned int arg = unknown, struct pt_regs * arg = 0xffff880045603c88, int arg = 0x9) # 3: [RSP: 0xffff880045603ad0, RIP: 0xffffffff81043bfb] no_context (struct pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0x10, long unsigned int arg = 0x0) # 4: [RSP: 0xffff880045603b20, RIP: 0xffffffff81043e85] __bad_area_nosemaphore (struct pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0x10, long unsigned int arg = 0x0, int arg = 0x30001) # 5: [RSP: 0xffff880045603b30, RIP: 0xffffffff81043f53] bad_area_nosemaphore (struct pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0x10, long unsigned int arg = 0x0) # 6: [RSP: 0xffff880045603c50, RIP: 0xffffffff810446b1] __do_page_fault (struct pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0x4b2, long unsigned int arg = 0x10) # 7: [RSP: 0xffff880045603c80, RIP: 0xffffffff8150383e] do_page_fault (struct pt_regs * arg = 0xffff880045603c88, long unsigned int arg = 0xffffffffffffffff) # 8: [RSP: 0xffff880045603d30, RIP: 0xffffffff81500bf5] page_fault () # 9: [RSP: 0xffff880045603d60, RIP: 0xffffffff81053916] enqueue_task (struct rq * arg = 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1) # 10: [RSP: 0xffff880045603d70, RIP: 0xffffffff81053953] activate_task (struct rq * arg = 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1) # 11: [RSP: 0xffff880045603de0, RIP: 0xffffffff810600c0] try_to_wake_up (struct task_struct * arg = 0xffff88082b83eae0, unsigned int arg = 0x3, int arg = 0x0) # 12: [RSP: 0xffff880045603df0, RIP: 0xffffffff810602c2] default_wake_function (wait_queue_t * arg = 0xffff8800456c3b40, unsigned int arg = 0x3, int arg = 0x0, void * arg = 0xc1416b9c06) # 13: [RSP: 0xffff880045603e40, RIP: 0xffffffff8104e369] __wake_up_common (wait_queue_head_t * arg = 0xffff880825e6a2a8, unsigned int arg = 0x3, int arg = 0x1, int arg = 0x0, void * arg = 0x0) # 14: [RSP: 0xffff880045603e70, RIP: 0xffffffff81053347] complete (struct completion * arg = 0xffff880825e6a2a0) # 15: [RSP: 0xffff880045603ed0, RIP: 0xffffffffa00a8f74] qla24xx_msix_default (int arg = 0x3e, void * arg = 0xffff880826bd55c0) # 16: [RSP: 0xffff880045603f20, RIP: 0xffffffff810dbb10] handle_IRQ_event (unsigned int arg = 0x3e, struct irqaction * arg = 0xffff880826bd5840) # 17: [RSP: 0xffff880045603f60, RIP: 0xffffffff810de29e] handle_edge_irq (unsigned int arg = 0x3e, struct irq_desc * arg = 0xffff880824563ec0) # 18: [RSP: 0xffff880045603f80, RIP: 0xffffffff8100df09] handle_irq (unsigned int arg = 0x3e, struct pt_regs * arg = 0xffffffff81a01da8) # 19: [RSP: 0xffff880045603fb0, RIP: 0xffffffff815060bc] do_IRQ (struct pt_regs * arg = 0xffffffff81a01da8) # 20: [RSP: 0xffffffff81a01e50, RIP: 0xffffffff8100ba53] common_interrupt () # 21: [RSP: 0xffffffff81a01ed0, RIP: 0xffffffff812cdc0e] intel_idle (struct cpuidle_device * arg = 0xffff88004561dcc0, struct cpuidle_state * arg = 0xffff88004561de50) # 22: [RSP: 0xffffffff81a01ef0, RIP: 0xffffffff81407997] cpuidle_idle_call () Some of the arguments are not accurate, but many of them are correct: bin> p/x *(( struct pt_regs * ) 0xffffffff81a01da8 ) $54 = { r15 = 0xffff8800456116a0, r14 = 0xc12ce6a080, r13 = 0x0, r12 = 0xffff8800456115a0, bp = 0xffffffff8100ba4e, bx = 0xffffffff81a01ec8, r11 = 0x0, r10 = 0xc26ede6d76, r9 = 0x6d, r8 = 0x5, ax = 0x0, cx = 0x0, dx = 0x116c, si = 0x0, di = 0x441165, orig_ax = 0xffffffffffffff9d, ip = 0xffffffff812cdc0e, cs = 0x10, flags = 0x206, sp = 0xffffffff81a01e58, ss = 0x18 } bin> p/x *((struct irq_desc * ) 0xffff880824563ec0) $55 = { irq = 0x3e, timer_rand_state = 0x0, kstat_irqs = 0xffff880825d7f6c0, irq_2_iommu = 0xffff880825e70880, handle_irq = 0xffffffff810de1c0, chip = 0xffffffff81a934e0, msi_desc = 0xffff880826bd56c0, handler_data = 0x0, chip_data = 0xffff880825e708a0, action = 0xffff880826bd5840, status = 0x1000100, depth = 0x0, wake_depth = 0x0, irq_count = 0x2cd, last_unhandled = 0x0, irqs_unhandled = 0x0, lock = { raw_lock = { slock = 0x5bc05bc } }, affinity = 0xffff880826a0f800, affinity_hint = 0x0, node = 0x0, pending_mask = 0xffff88082b765a00, threads_active = { counter = 0x0 }, wait_for_threads = { lock = { raw_lock = { slock = 0x0 } }, task_list = { next = 0xffff880824563f60, prev = 0xffff880824563f60 } }, dir = 0xffff880824563d40, name = 0xffffffff8177f1ce } =======> CPU #6 bin> bt PID: 0 TASK: ffff88082b83eae0 CPU: 6 COMMAND: "swapper" #0 [ffff8800456c7e90] crash_nmi_callback at ffffffff81029df6 #1 [ffff8800456c7ea0] notifier_call_chain at ffffffff815038f5 #2 [ffff8800456c7ee0] atomic_notifier_call_chain at ffffffff8150395a #3 [ffff8800456c7ef0] notify_die at ffffffff810981ee #4 [ffff8800456c7f20] do_nmi at ffffffff81501573 #5 [ffff8800456c7f50] nmi at ffffffff81500e80 [exception RIP: _spin_lock_irq+37] RIP: ffffffff81500615 RSP: ffff8800456c3940 RFLAGS: 00000097 RAX: 00000000000008a4 RBX: ffff8800456d6680 RCX: 000000000000c14e RDX: 00000000000008a3 RSI: 0000000000000001 RDI: ffff8800456d6680 RBP: ffff8800456c3940 R8: 0000000000013679 R9: 00000000fffffffa R10: 0000000000000002 R11: 0000000000000002 R12: ffff8800456d6680 R13: 0000000000000001 R14: 00000000ffffffff R15: 00000000ffffffff ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- <NMI exception stack> --- #6 [ffff8800456c3940] _spin_lock_irq at ffffffff81500615 #7 [ffff8800456c3948] dequeue_task_idle at ffffffff81053f19 #8 [ffff8800456c3968] dequeue_task at ffffffff8105385e #9 [ffff8800456c3998] deactivate_task at ffffffff810538a3 #10 [ffff8800456c39a8] thread_return at ffffffff814fded9 #11 [ffff8800456c3a68] schedule_timeout at ffffffff814febf2 #12 [ffff8800456c3b18] wait_for_common at ffffffff814fe8f3 #13 [ffff8800456c3ba8] wait_for_completion_timeout at ffffffff814fe9e3 #14 [ffff8800456c3bb8] qla2x00_mailbox_command at ffffffffa0098d1a [qla2xxx] #15 [ffff8800456c3c68] qla2x00_issue_iocb_timeout at ffffffffa009cc3b [qla2xxx] #16 [ffff8800456c3cf8] qla2x00_issue_iocb at ffffffffa009ccd4 [qla2xxx] #17 [ffff8800456c3d08] qla24xx_abort_command at ffffffffa009f5b8 [qla2xxx] #18 [ffff8800456c3d68] qla24xx_bsg_timeout at ffffffffa00c0ba6 [qla2xxx] #19 [ffff8800456c3da8] fc_bsg_job_timeout at ffffffffa00718f7 [scsi_transport_fc] #20 [ffff8800456c3dd8] blk_rq_timed_out at ffffffff8125dccb #21 [ffff8800456c3df8] blk_rq_timed_out_timer at ffffffff8125df88 #22 [ffff8800456c3e48] run_timer_softirq at ffffffff8107e927 #23 [ffff8800456c3ed8] __do_softirq at ffffffff81073f51 #24 [ffff8800456c3f48] call_softirq at ffffffff8100c24c #25 [ffff8800456c3f60] do_softirq at ffffffff8100de85 #26 [ffff8800456c3f80] irq_exit at ffffffff81073d35 #27 [ffff8800456c3f90] smp_apic_timer_interrupt at ffffffff815061b0 #28 [ffff8800456c3fb0] apic_timer_interrupt at ffffffff8100bc13 --- <IRQ stack> --- #29 [ffff88082b847db8] apic_timer_interrupt at ffffffff8100bc13 [exception RIP: intel_idle+222] RIP: ffffffff812cdc0e RSP: ffff88082b847e68 RFLAGS: 00000206 RAX: 0000000000000000 RBX: ffff88082b847ed8 RCX: 0000000000000000 RDX: 0000000000000f69 RSI: 0000000000000000 RDI: 00000000003c35fa RBP: ffffffff8100bc0e R8: 0000000000000005 R9: 000000000000006d R10: 000000c130c95aec R11: 0000000000000000 R12: ffff8800456d15a0 R13: 0000000000000000 R14: 000000c12d23a980 R15: ffff8800456d16a0 ORIG_RAX: ffffffffffffff10 CS: 0010 SS: 0018 #30 [ffff88082b847ee0] cpuidle_idle_call at ffffffff81407997 #31 [ffff88082b847f00] cpu_idle at ffffffff81009e06 bin> bt -H # 0: [RSP: 0xffff8800456c7ea0, RIP: 0xffffffff81029df6] crash_nmi_callback (struct notifier_block * arg = 0xffffffff81aa5260, long unsigned int arg = 0xc, void * arg = 0xffff8800456c7ef8) # 1: [RSP: 0xffff8800456c7ee0, RIP: 0xffffffff815038f5] notifier_call_chain (struct notifier_block ** arg = 0xffffffff81ea87a8, long unsigned int arg = 0xc, void * arg = 0xffff8800456c7ef8, int arg = 0xffffffff, int * arg = 0x0) # 2: [RSP: 0xffff8800456c7ef0, RIP: 0xffffffff8150395a] atomic_notifier_call_chain (struct atomic_notifier_head * arg = 0xffffffff81ea87a0, long unsigned int arg = 0xc, void * arg = 0xffff8800456c7ef8) # 3: [RSP: 0xffff8800456c7f20, RIP: 0xffffffff810981ee] notify_die (enum die_val arg = 0xc, const char * arg = 0xffffffff8177d9a2, struct pt_regs * arg = 0xffff8800456c7f58, long int arg = 0x0, int arg = 0x2, int arg = 0x2) # 4: [RSP: 0xffff8800456c7f50, RIP: 0xffffffff81501573] do_nmi (struct pt_regs * arg = 0xffff8800456c7f58, long int arg = 0xffffffffffffffff) # 5: [RSP: 0xffff8800456c8000, RIP: 0xffffffff81500e80] nmi () # 6: [RSP: 0xffff8800456c3948, RIP: 0xffffffff81500615] _spin_lock_irq (spinlock_t * arg = 0xffff8800456d6680) # 7: [RSP: 0xffff8800456c3968, RIP: 0xffffffff81053f19] dequeue_task_idle (struct rq * arg = 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1) # 8: [RSP: 0xffff8800456c3998, RIP: 0xffffffff8105385e] dequeue_task (struct rq * arg = 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1) # 9: [RSP: 0xffff8800456c39a8, RIP: 0xffffffff810538a3] deactivate_task (struct rq * arg = 0xffff8800456d6680, struct task_struct * arg = 0xffff88082b83eae0, int arg = 0x1) # 10: [RSP: 0xffff8800456c3a68, RIP: 0xffffffff814fded9] thread_return () # 11: [RSP: 0xffff8800456c3b18, RIP: 0xffffffff814febf2] schedule_timeout (long int arg = 0x7530) # 12: [RSP: 0xffff8800456c3ba8, RIP: 0xffffffff814fe8f3] wait_for_common (struct completion * arg = 0xffff880825e6a2a0, long int arg = 0x7530, int arg = 0x2) # 13: [RSP: 0xffff8800456c3bb8, RIP: 0xffffffff814fe9e3] wait_for_completion_timeout (struct completion * arg = 0xffff880825e6a2a0, long unsigned int arg = 0x7530) # 14: [RSP: 0xffff8800456c3c68, RIP: 0xffffffffa0098d1a] qla2x00_mailbox_command (scsi_qla_host_t * arg = 0xffff880824562de0, mbx_cmd_t * arg = 0xffff8800456c3c70) # 15: [RSP: 0xffff8800456c3cf8, RIP: 0xffffffffa009cc3b] qla2x00_issue_iocb_timeout (scsi_qla_host_t * arg = 0xffff880824562de0, void * arg = 0xffff880824561100, dma_addr_t arg = 0x824561100, size_t arg = 0x0, uint32_t arg = 0x1e) # 16: [RSP: 0xffff8800456c3d08, RIP: 0xffffffffa009ccd4] qla2x00_issue_iocb (scsi_qla_host_t * arg = 0xffff880824562de0, void * arg = 0xffff880824561100, dma_addr_t arg = 0x824561100, size_t arg = 0x0) # 17: [RSP: 0xffff8800456c3d68, RIP: 0xffffffffa009f5b8] qla24xx_abort_command (srb_t * arg = 0xffff8808145db480) # 18: [RSP: 0xffff8800456c3da8, RIP: 0xffffffffa00c0ba6] qla24xx_bsg_timeout (struct fc_bsg_job * arg = 0xffff8808268a2740) # 19: [RSP: 0xffff8800456c3dd8, RIP: 0xffffffffa00718f7] fc_bsg_job_timeout (struct request * arg = 0xffff88082446b360) # 20: [RSP: 0xffff8800456c3df8, RIP: 0xffffffff8125dccb] blk_rq_timed_out (struct request * arg = 0xffff88082446b360) # 21: [RSP: 0xffff8800456c3e48, RIP: 0xffffffff8125df88] blk_rq_timed_out_timer (long unsigned int arg = 0xffff88082477cea0) # 22: [RSP: 0xffff8800456c3ed8, RIP: 0xffffffff8107e927] run_timer_softirq (struct softirq_action * arg = 0xffffffff81a83088) # 23: [RSP: 0xffff8800456c3f48, RIP: 0xffffffff81073f51] __do_softirq () # 24: [RSP: 0xffff8800456c3f60, RIP: 0xffffffff8100c24c] call_softirq () # 25: [RSP: 0xffff8800456c3f80, RIP: 0xffffffff8100de85] do_softirq () # 26: [RSP: 0xffff8800456c3f90, RIP: 0xffffffff81073d35] irq_exit () # 27: [RSP: 0xffff8800456c3fb0, RIP: 0xffffffff815061b0] smp_apic_timer_interrupt (struct pt_regs * arg = 0xffff88082b847db8) # 28: [RSP: 0xffff88082b847e60, RIP: 0xffffffff8100bc13] apic_timer_interrupt () # 29: [RSP: 0xffff88082b847ee0, RIP: 0xffffffff812cdc0e] intel_idle (struct cpuidle_device * arg = 0xffff8800456ddcc0, struct cpuidle_state * arg = 0xffff8800456dde50) # 30: [RSP: 0xffff88082b847f00, RIP: 0xffffffff81407997] cpuidle_idle_call () Please feel free to submit any found issues via email or github. -- Crash-utility mailing list Crash-utility@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/crash-utility
