Hi,
On Thu, Jan 11, 2007 at 04:22:52PM -0500, Bino.Sebastian@xxxxxxxxxx wrote:
> This patch is present in upstream and is also present
> in 2.6.20. So this is a new issue.
What was the patch last time around?
It seems I'm seeing this more often than expected. The first time,
the system spewed the softlockup error, but then recovered after
a few minutes. This time, even after an hour, the system remained
hung. It was pingable, but the console, and all ssh sessions
were unresponsive.
After hitting the little yellow button, I got a stack trace
(below) in _spin_unlock_irqrestore, which makes me think that
perhaps the system was being flooded with irq's. I'll try
to investigate further tommorrow.
--linas
Background:
kernel 2.6.20-rc4
IBM Power4 pSeries (630)
lpfc scsi (Emulex)
chsysstate -r sys -n io-raiders -o reset
io-raiders:~ # cpu 0x0: Vector: 100 (System Reset) at [c00000003ff69520]
pc: c00000000023d794: ._raw_spin_unlock+0xb4/0xd4
lr: c00000000046d5ac: ._spin_unlock_irqrestore+0x18/0x3c
sp: c00000003ff697a0
msr: 9000000000009032
current = 0xc0000000043e21f0
paca = 0xc000000000674080
pid = 1123, comm = kblockd/0
enter ? for help
[c00000003ff69820] c00000000046d5ac ._spin_unlock_irqrestore+0x18/0x3c
[c00000003ff698b0] c00000000021bbe0 .blk_run_queue+0xc8/0xec
[c00000003ff69950] c000000000320728 .scsi_run_queue+0x248/0x278
[c00000003ff69a00] c000000000321948 .scsi_queue_insert+0x88/0xa8
[c00000003ff69a90] c00000000031bc34 .scsi_dispatch_cmd+0x2b8/0x2e4
[c00000003ff69b30] c000000000322804 .scsi_request_fn+0x2c4/0x3c0
[c00000003ff69be0] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff69c60] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff69d00] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff69d90] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff69e40] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff69ec0] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff69f60] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff69ff0] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6a0a0] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6a120] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6a1c0] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6a250] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6a300] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6a380] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6a420] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6a4b0] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6a560] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6a5e0] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6a680] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6a710] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6a7c0] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6a840] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6a8e0] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6a970] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6aa20] c00000000021bbac .blk_run_queue+0x94/0xec
[c00000003ff6aac0] c000000000320728 .scsi_run_queue+0x248/0x278
[c00000003ff6ab70] c000000000321948 .scsi_queue_insert+0x88/0xa8
[c00000003ff6ac00] c00000000031bc34 .scsi_dispatch_cmd+0x2b8/0x2e4
[c00000003ff6aca0] c000000000322804 .scsi_request_fn+0x2c4/0x3c0
[c00000003ff6ad50] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6add0] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6ae70] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6af00] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6afb0] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6b030] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6b0d0] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6b160] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6b210] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6b290] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6b330] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6b3c0] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6b470] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6b4f0] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6b590] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6b620] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6b6d0] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6b750] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6b7f0] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6b880] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6b930] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6b9b0] c000000000216d6c .elv_insert+0x240/0x268
[c00000003ff6ba50] c00000000021a25c .blk_requeue_request+0x38/0x54
[c00000003ff6bae0] c000000000322864 .scsi_request_fn+0x324/0x3c0
[c00000003ff6bb90] c00000000021ae30 .__generic_unplug_device+0x54/0x6c
[c00000003ff6bc10] c00000000021b3b0 .generic_unplug_device+0x30/0x50
[c00000003ff6bca0] c0000000002177d0 .blk_unplug_work+0x34/0x48
[c00000003ff6bd20] c000000000072214 .run_workqueue+0xe8/0x1d4
[c00000003ff6bdc0] c00000000007305c .worker_thread+0x148/0x1b8
0:mon>
0:mon> r
R00 = 0000000000000000 R16 = 4000000003a10000
R01 = c00000003ff697a0 R17 = c00000000055e0b0
R02 = c0000000007b0128 R18 = 0000000000000000
R03 = c00000000339b330 R19 = 0000000000000000
R04 = 0000000000000001 R20 = 0000000004041e98
R05 = 0000000000000000 R21 = c000000000631e98
R06 = 0000000000000000 R22 = 0000000004042108
R07 = 0000000000000000 R23 = c000000000632108
R08 = c00000003ffe5f90 R24 = c00000000339b198
R09 = 0000000000000000 R25 = c00000002f440438
R10 = c00000003ffe5fa0 R26 = c00000002f440338
R11 = c0000003feee8cd8 R27 = 0000000000000001
R12 = d0000000008d2480 R28 = 0000000000000040
R13 = c000000000674080 R29 = 0000000000000001
R14 = 0000000000000000 R30 = c0000000006da320
R15 = c00000000055f688 R31 = c00000000339b330
pc = c00000000023d794 ._raw_spin_unlock+0xb4/0xd4
lr = c00000000046d5ac ._spin_unlock_irqrestore+0x18/0x3c
msr = 9000000000009032 cr = 24002082
ctr = c0000000002240c0 xer = 0000000000000000 trap = 100
0:mon>
0:mon> di c00000000023d6e0
c00000000023d6e0 7c0802a6 mflr r0
c00000000023d6e4 fbc1fff0 std r30,-16(r1)
c00000000023d6e8 fbe1fff8 std r31,-8(r1)
c00000000023d6ec ebc2e2b8 ld r30,-7496(r2)
c00000000023d6f0 7c7f1b78 mr r31,r3
c00000000023d6f4 f8010010 std r0,16(r1)
c00000000023d6f8 f821ff81 stdu r1,-128(r1)
c00000000023d6fc 3c00dead lis r0,-8531
c00000000023d700 60004ead ori r0,r0,20141
c00000000023d704 81230004 lwz r9,4(r3)
c00000000023d708 7f890000 cmpw cr7,r9,r0
c00000000023d70c 41be000c beq cr7,c00000000023d718 # ._raw_spin_unlock+0x38/0xd4
c00000000023d710 e89e8008 ld r4,-32760(r30)
c00000000023d714 4bfffef9 bl c00000000023d60c # .spin_bug+0x0/0xd4
c00000000023d718 801f0000 lwz r0,0(r31)
c00000000023d71c 2fa00000 cmpdi cr7,r0,0
0:mon>
c00000000023d720 40be0010 bne cr7,c00000000023d730 # ._raw_spin_unlock+0x50/0xd4
c00000000023d724 e89e8038 ld r4,-32712(r30)
c00000000023d728 7fe3fb78 mr r3,r31
c00000000023d72c 4bfffee1 bl c00000000023d60c # .spin_bug+0x0/0xd4
c00000000023d730 e80d01a0 ld r0,416(r13)
c00000000023d734 e93f0010 ld r9,16(r31)
c00000000023d738 7fa90000 cmpd cr7,r9,r0
c00000000023d73c 41be0010 beq cr7,c00000000023d74c # ._raw_spin_unlock+0x6c/0xd4
c00000000023d740 e89e8010 ld r4,-32752(r30)
c00000000023d744 7fe3fb78 mr r3,r31
c00000000023d748 4bfffec5 bl c00000000023d60c # .spin_bug+0x0/0xd4
c00000000023d74c a00d000a lhz r0,10(r13)
c00000000023d750 813f0008 lwz r9,8(r31)
c00000000023d754 7f890000 cmpw cr7,r9,r0
c00000000023d758 41be0010 beq cr7,c00000000023d768 # ._raw_spin_unlock+0x88/0xd4
c00000000023d75c e89e8018 ld r4,-32744(r30)
0:mon>
c00000000023d760 7fe3fb78 mr r3,r31
c00000000023d764 4bfffea9 bl c00000000023d60c # .spin_bug+0x0/0xd4
c00000000023d768 3800ffff li r0,-1
c00000000023d76c 901f0008 stw r0,8(r31)
c00000000023d770 f81f0010 std r0,16(r31)
c00000000023d774 60000000 nop
c00000000023d778 880d01ca lbz r0,458(r13)
c00000000023d77c 2f800000 cmpwi cr7,r0,0
c00000000023d780 41be0010 beq cr7,c00000000023d790 # ._raw_spin_unlock+0xb0/0xd4
c00000000023d784 7c0004ac sync
c00000000023d788 38000000 li r0,0
c00000000023d78c 980d01ca stb r0,458(r13)
c00000000023d790 7c2004ac lwsync
c00000000023d794 38000000 li r0,0
c00000000023d798 38210080 addi r1,r1,128
c00000000023d79c 901f0000 stw r0,0(r31)
0:mon>
-
To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
