Hi James,
With the next-20190301 kernel tree I ran into the lockdep complaint
shown below during shutdown. Can you have a look?
Thanks,
Bart.
=====================================================
WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected
5.0.0-rc8-next-20190301-dbg+ #1 Not tainted
-----------------------------------------------------
shutdown/1 [HC0[0]:SC0[0]:HE0:SE1] is trying to acquire:
000000002fea93ea (&(&qp->io_buf_list_put_lock)->rlock){+.+.}, at: lpfc_io_free+0xa2/0x360 [lpfc]
and this task is already holding:
00000000cfd51f54 (&(&phba->hbalock)->rlock){..-.}, at: lpfc_io_free+0x30/0x360 [lpfc]
which would create a new lock dependency:
(&(&phba->hbalock)->rlock){..-.} -> (&(&qp->io_buf_list_put_lock)->rlock){+.+.}
but this new dependency connects a SOFTIRQ-irq-safe lock:
(&(&phba->hbalock)->rlock){..-.}
... which became SOFTIRQ-irq-safe at:
lock_acquire+0xe3/0x200
_raw_spin_lock_irq+0x38/0x50
lpfc_sli_check_eratt.part.48+0x2f/0x570 [lpfc]
lpfc_poll_eratt+0xef/0x140 [lpfc]
call_timer_fn+0x10b/0x3f0
run_timer_softirq+0x3f1/0x960
__do_softirq+0x12d/0x5b7
irq_exit+0xdd/0x100
smp_apic_timer_interrupt+0xe4/0x340
apic_timer_interrupt+0xf/0x20
default_idle+0x23/0x1f0
arch_cpu_idle+0x15/0x20
default_idle_call+0x35/0x40
do_idle+0x2bb/0x2e0
cpu_startup_entry+0x1d/0x20
start_secondary+0x245/0x340
secondary_startup_64+0xa4/0xb0
to a SOFTIRQ-irq-unsafe lock:
(&(&qp->io_buf_list_put_lock)->rlock){+.+.}
... which became SOFTIRQ-irq-unsafe at:
...
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_buf_replenish+0x103/0x260 [lpfc]
lpfc_sli4_post_io_sgl_list+0xa80/0xba0 [lpfc]
lpfc_new_io_buf+0x370/0x4f0 [lpfc]
lpfc_sli4_hba_setup+0x232d/0x2570 [lpfc]
lpfc_pci_probe_one.cold.93+0x1816/0x1b1e [lpfc]
local_pci_probe+0x7a/0xc0
pci_device_probe+0x2f8/0x310
really_probe+0x182/0x540
driver_probe_device+0x118/0x180
device_driver_attach+0x8d/0xa0
__driver_attach+0xba/0x1a0
bus_for_each_dev+0xfd/0x160
driver_attach+0x2b/0x30
bus_add_driver+0x23d/0x2f0
driver_register+0xdc/0x160
__pci_register_driver+0xda/0xf0
wait_transaction_locked+0xf7/0x1b0 [jbd2]
do_one_initcall+0xc5/0x3ce
do_init_module+0x108/0x360
load_module+0x36eb/0x3b80
__do_sys_finit_module+0x133/0x1c0
__x64_sys_finit_module+0x43/0x50
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
other info that might help us debug this:
Possible interrupt unsafe locking scenario:
CPU0 CPU1
---- ----
lock(&(&qp->io_buf_list_put_lock)->rlock);
local_irq_disable();
lock(&(&phba->hbalock)->rlock);
lock(&(&qp->io_buf_list_put_lock)->rlock);
<Interrupt>
lock(&(&phba->hbalock)->rlock);
*** DEADLOCK ***
4 locks held by shutdown/1:
#0: 00000000493e5481 (system_transition_mutex){+.+.}, at: __do_sys_reboot+0x112/0x2a0
#1: 00000000a0e71590 (&dev->mutex){....}, at: device_shutdown+0x105/0x2c0
#2: 000000009e6eba39 (&dev->mutex){....}, at: device_shutdown+0x113/0x2c0
#3: 00000000cfd51f54 (&(&phba->hbalock)->rlock){..-.}, at: lpfc_io_free+0x30/0x360 [lpfc]
the dependencies between SOFTIRQ-irq-safe lock and the holding lock:
-> (&(&phba->hbalock)->rlock){..-.} ops: 45890 {
IN-SOFTIRQ-W at:
lock_acquire+0xe3/0x200
_raw_spin_lock_irq+0x38/0x50
lpfc_sli_check_eratt.part.48+0x2f/0x570 [lpfc]
lpfc_poll_eratt+0xef/0x140 [lpfc]
call_timer_fn+0x10b/0x3f0
run_timer_softirq+0x3f1/0x960
__do_softirq+0x12d/0x5b7
irq_exit+0xdd/0x100
smp_apic_timer_interrupt+0xe4/0x340
apic_timer_interrupt+0xf/0x20
default_idle+0x23/0x1f0
arch_cpu_idle+0x15/0x20
default_idle_call+0x35/0x40
do_idle+0x2bb/0x2e0
cpu_startup_entry+0x1d/0x20
start_secondary+0x245/0x340
secondary_startup_64+0xa4/0xb0
INITIAL USE at:
lock_acquire+0xe3/0x200
_raw_spin_lock_irqsave+0x3d/0x60
lpfc_sli4_post_sync_mbox+0x33/0x560 [lpfc]
lpfc_sli_issue_mbox_s4+0x580/0x6d0 [lpfc]
lpfc_sli_issue_mbox+0x32/0x40 [lpfc]
lpfc_sli4_read_config+0x64/0xa90 [lpfc]
lpfc_pci_probe_one+0x188d/0x2340 [lpfc]
local_pci_probe+0x7a/0xc0
pci_device_probe+0x2f8/0x310
really_probe+0x182/0x540
driver_probe_device+0x118/0x180
device_driver_attach+0x8d/0xa0
__driver_attach+0xba/0x1a0
bus_for_each_dev+0xfd/0x160
driver_attach+0x2b/0x30
bus_add_driver+0x23d/0x2f0
driver_register+0xdc/0x160
__pci_register_driver+0xda/0xf0
wait_transaction_locked+0xf7/0x1b0 [jbd2]
do_one_initcall+0xc5/0x3ce
do_init_module+0x108/0x360
load_module+0x36eb/0x3b80
__do_sys_finit_module+0x133/0x1c0
__x64_sys_finit_module+0x43/0x50
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
}
... key at: [<ffffffffa0407d00>] __key.61834+0x0/0xfffffffffff84300 [lpfc]
... acquired at:
__lock_acquire+0x11ed/0x1b60
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_free+0xa2/0x360 [lpfc]
lpfc_pci_remove_one+0x832/0xf10 [lpfc]
pci_device_shutdown+0x50/0x90
device_shutdown+0x1ef/0x2c0
kernel_restart_prepare+0x42/0x50
kernel_restart+0x12/0x30
__do_sys_reboot+0x145/0x2a0
__x64_sys_reboot+0x55/0x60
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
the dependencies between the lock to be acquired
and SOFTIRQ-irq-unsafe lock:
-> (&(&qp->io_buf_list_put_lock)->rlock){+.+.} ops: 11845 {
HARDIRQ-ON-W at:
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_buf_replenish+0x103/0x260 [lpfc]
lpfc_sli4_post_io_sgl_list+0xa80/0xba0 [lpfc]
lpfc_new_io_buf+0x370/0x4f0 [lpfc]
lpfc_sli4_hba_setup+0x232d/0x2570 [lpfc]
lpfc_pci_probe_one.cold.93+0x1816/0x1b1e [lpfc]
local_pci_probe+0x7a/0xc0
pci_device_probe+0x2f8/0x310
really_probe+0x182/0x540
driver_probe_device+0x118/0x180
device_driver_attach+0x8d/0xa0
__driver_attach+0xba/0x1a0
bus_for_each_dev+0xfd/0x160
driver_attach+0x2b/0x30
bus_add_driver+0x23d/0x2f0
driver_register+0xdc/0x160
__pci_register_driver+0xda/0xf0
wait_transaction_locked+0xf7/0x1b0 [jbd2]
do_one_initcall+0xc5/0x3ce
do_init_module+0x108/0x360
load_module+0x36eb/0x3b80
__do_sys_finit_module+0x133/0x1c0
__x64_sys_finit_module+0x43/0x50
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
SOFTIRQ-ON-W at:
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_buf_replenish+0x103/0x260 [lpfc]
lpfc_sli4_post_io_sgl_list+0xa80/0xba0 [lpfc]
lpfc_new_io_buf+0x370/0x4f0 [lpfc]
lpfc_sli4_hba_setup+0x232d/0x2570 [lpfc]
lpfc_pci_probe_one.cold.93+0x1816/0x1b1e [lpfc]
local_pci_probe+0x7a/0xc0
pci_device_probe+0x2f8/0x310
really_probe+0x182/0x540
driver_probe_device+0x118/0x180
device_driver_attach+0x8d/0xa0
__driver_attach+0xba/0x1a0
bus_for_each_dev+0xfd/0x160
driver_attach+0x2b/0x30
bus_add_driver+0x23d/0x2f0
driver_register+0xdc/0x160
__pci_register_driver+0xda/0xf0
wait_transaction_locked+0xf7/0x1b0 [jbd2]
do_one_initcall+0xc5/0x3ce
do_init_module+0x108/0x360
load_module+0x36eb/0x3b80
__do_sys_finit_module+0x133/0x1c0
__x64_sys_finit_module+0x43/0x50
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
INITIAL USE at:
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_buf_flush+0x12e/0x590 [lpfc]
lpfc_sli4_io_sgl_update+0x15e/0x3e0 [lpfc]
lpfc_sli4_hba_setup+0x1d45/0x2570 [lpfc]
lpfc_pci_probe_one.cold.93+0x1816/0x1b1e [lpfc]
local_pci_probe+0x7a/0xc0
pci_device_probe+0x2f8/0x310
really_probe+0x182/0x540
driver_probe_device+0x118/0x180
device_driver_attach+0x8d/0xa0
__driver_attach+0xba/0x1a0
bus_for_each_dev+0xfd/0x160
driver_attach+0x2b/0x30
bus_add_driver+0x23d/0x2f0
driver_register+0xdc/0x160
__pci_register_driver+0xda/0xf0
wait_transaction_locked+0xf7/0x1b0 [jbd2]
do_one_initcall+0xc5/0x3ce
do_init_module+0x108/0x360
load_module+0x36eb/0x3b80
__do_sys_finit_module+0x133/0x1c0
__x64_sys_finit_module+0x43/0x50
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
}
... key at: [<ffffffffa0407f80>] __key.62303+0x0/0xfffffffffff84080 [lpfc]
... acquired at:
__lock_acquire+0x11ed/0x1b60
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_free+0xa2/0x360 [lpfc]
lpfc_pci_remove_one+0x832/0xf10 [lpfc]
pci_device_shutdown+0x50/0x90
device_shutdown+0x1ef/0x2c0
kernel_restart_prepare+0x42/0x50
kernel_restart+0x12/0x30
__do_sys_reboot+0x145/0x2a0
__x64_sys_reboot+0x55/0x60
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
stack backtrace:
CPU: 4 PID: 1 Comm: shutdown Not tainted 5.0.0-rc8-next-20190301-dbg+ #1
Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011
Call Trace:
dump_stack+0x86/0xca
check_usage.cold.59+0x473/0x563
check_prev_add.constprop.43+0x241/0x1170
__lock_acquire+0x11ed/0x1b60
lock_acquire+0xe3/0x200
_raw_spin_lock+0x32/0x50
lpfc_io_free+0xa2/0x360 [lpfc]
lpfc_pci_remove_one+0x832/0xf10 [lpfc]
pci_device_shutdown+0x50/0x90
device_shutdown+0x1ef/0x2c0
kernel_restart_prepare+0x42/0x50
kernel_restart+0x12/0x30
__do_sys_reboot+0x145/0x2a0
__x64_sys_reboot+0x55/0x60
do_syscall_64+0x77/0x220
entry_SYSCALL_64_after_hwframe+0x49/0xbe
reboot: Restarting system
reboot: machine restart
