On 1/11/21 5:37 PM, Keith Busch wrote:
On Mon, Jan 11, 2021 at 02:39:20PM +0100, Hinko Kocevar wrote:
Testing this patch a bit more (without the 5/5) resulted in the same CPU
lockup:
watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [irq/122-aerdrv:128]
as I initially reported with the 5/5 of this patch included.
It seems more infrequent, though. For example, after reboot this is not
observed and the recovery process is successful, whereas when 5/5 is also
used every recovery resulted in CPU lockup.
I am assuming this soft lockup is still when restoring the downstream
port's virtual channel capability. Your initial sighting indicates that
it doesn't appear to be a deadlock, but the stack trace never existed
pci_restore_vc_state() either. I did not find any obvious issues here
just from code inspection, so if you could try applying the following
patch and send the kernel messages output, that would help.
---
diff --git a/drivers/pci/vc.c b/drivers/pci/vc.c
index 5fc59ac31145..4834af7eb582 100644
--- a/drivers/pci/vc.c
+++ b/drivers/pci/vc.c
@@ -28,6 +28,7 @@ static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos,
{
int i;
+ pci_warn(dev, "%s: pos:%d dwords:%d\n", __func__, pos, dwords);
for (i = 0; i < dwords; i++, buf++) {
if (save)
pci_read_config_dword(dev, pos + (i * 4), buf);
@@ -110,6 +111,8 @@ static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
if (!pci_is_pcie(dev) || !pcie_downstream_port(dev))
return;
+ pci_warn(dev, "%s: pos:%d res:%d\n", __func__, pos, res);
+
ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
@@ -165,6 +168,8 @@ static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
if (link && !pci_wait_for_pending(link, status_pos2,
PCI_VC_RES_STATUS_NEGO))
pci_err(link, "VC%d negotiation stuck pending\n", id);
+
+ pci_warn(dev, "%s: pos:%d res:%d return\n", __func__, pos, res);
}
/**
@@ -190,6 +195,7 @@ static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
int i, len = 0;
u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL;
+ pci_warn(dev, "%s: buf:%d pos:%d\n", __func__, buf != NULL, pos);
/* Sanity check buffer size for save/restore */
if (buf && save_state->cap.size !=
pci_vc_do_save_buffer(dev, pos, NULL, save)) {
@@ -278,6 +284,8 @@ static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
pci_read_config_dword(dev, pos + PCI_VC_RES_CAP +
(i * PCI_CAP_VC_PER_VC_SIZEOF), &cap);
parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16;
+ pci_warn(dev, "%s: i:%d evcc:%d parb_offset:%d\n", __func__, i,
+ evcc, parb_offset);
if (parb_offset) {
int size, parb_phases = 0;
@@ -332,6 +340,7 @@ static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
len += 4;
}
+ pci_warn(dev, "%s: len:%d\n", __func__, len);
return buf ? 0 : len;
}
@@ -399,6 +408,7 @@ void pci_restore_vc_state(struct pci_dev *dev)
if (!save_state || !pos)
continue;
+ pci_warn(dev, "%s: i:%d pos:%d\n", __func__, i, pos);
pci_vc_do_save_buffer(dev, pos, save_state, false);
}
}
--
Attached are the messages.
Thanks!
...
[ 64.649399] xdma: loading out-of-tree module taints kernel.
[ 64.656255] xdma:xdma_mod_init: Xilinx XDMA Reference Driver xdma v2020.1.8
[ 64.663324] xdma:xdma_mod_init: desc_blen_max: 0xfffffff/268435455, timeout: h2c 10 c2h 10 sec.
[ 64.672189] xdma:xdma_device_open: xdma device 0000:08:00.0, 0x00000000591acbe0.
[ 64.680346] xdma:map_single_bar: BAR0 at 0xde000000 mapped at 0x000000000fdb12a5, length=16777216(/16777216)
[ 64.690290] xdma:map_single_bar: BAR1 at 0xdf000000 mapped at 0x000000005fe214c9, length=65536(/65536)
[ 64.699702] xdma:map_bars: config bar 1, pos 1.
[ 64.704339] xdma:identify_bars: 2 BARs: config 1, user 0, bypass -1.
[ 64.711245] xdma:pci_keep_intx_enabled: 0000:08:00.0: clear INTX_DISABLE, 0x407 -> 0x7.
[ 64.719369] xdma:irq_msix_channel_setup: engine 0-H2C0-MM, irq#169.
[ 64.725751] xdma:irq_msix_channel_setup: engine 0-C2H0-MM, irq#170.
[ 64.732134] xdma:irq_msix_user_setup: 0-USR-0, IRQ#171 with 0x0000000016487e14
[ 64.739479] xdma:irq_msix_user_setup: 0-USR-1, IRQ#172 with 0x0000000017b744b7
[ 64.746813] xdma:irq_msix_user_setup: 0-USR-2, IRQ#173 with 0x000000007491af44
[ 64.754149] xdma:irq_msix_user_setup: 0-USR-3, IRQ#174 with 0x000000009e92f5ff
[ 64.761484] xdma:irq_msix_user_setup: 0-USR-4, IRQ#175 with 0x0000000056e09b58
[ 64.768819] xdma:irq_msix_user_setup: 0-USR-5, IRQ#176 with 0x00000000af6d01f5
[ 64.776159] xdma:irq_msix_user_setup: 0-USR-6, IRQ#177 with 0x000000002ef15b69
[ 64.783493] xdma:irq_msix_user_setup: 0-USR-7, IRQ#178 with 0x0000000099fd5ab6
[ 64.790832] xdma:irq_msix_user_setup: 0-USR-8, IRQ#179 with 0x0000000014654474
[ 64.798167] xdma:irq_msix_user_setup: 0-USR-9, IRQ#180 with 0x00000000963ca534
[ 64.805504] xdma:irq_msix_user_setup: 0-USR-10, IRQ#181 with 0x0000000086853299
[ 64.812928] xdma:irq_msix_user_setup: 0-USR-11, IRQ#182 with 0x00000000d0d53381
[ 64.820354] xdma:irq_msix_user_setup: 0-USR-12, IRQ#183 with 0x000000008ce02f44
[ 64.827776] xdma:irq_msix_user_setup: 0-USR-13, IRQ#184 with 0x00000000bc4b16b8
[ 64.835200] xdma:irq_msix_user_setup: 0-USR-14, IRQ#185 with 0x0000000088677e88
[ 64.842623] xdma:irq_msix_user_setup: 0-USR-15, IRQ#186 with 0x000000000d9849f6
[ 64.850044] xdma:probe_one: 0000:08:00.0 xdma0, pdev 0x00000000591acbe0, xdev 0x00000000c77a5002, 0x00000000947398f0, usr 16, ch 1,1.
[ 64.864139] xdma:cdev_xvc_init: xcdev 0x000000004eda4704, bar 0, offset 0x40000.
[ 68.523443] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 68.541024] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 68.547489] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 68.554208] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 68.560929] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.568603] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.574632] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.582305] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.588338] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 68.597139] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 68.605946] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 68.615568] pcieport 0000:00:01.1: PME# disabled
[ 68.739042] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 68.747225] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 68.755222] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 68.763310] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 68.771310] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 68.778974] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 68.786628] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 68.794629] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 68.802379] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 68.810298] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 68.818315] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 68.825980] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 68.833651] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 68.841312] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 68.849061] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 68.856719] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 68.864817] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 68.871433] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 68.878049] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.885620] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.891555] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.899128] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.905178] pcieport 0000:00:01.1: PME# enabled
[ 68.921051] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 68.927905] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 68.946078] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 68.952545] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 68.959265] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 68.965984] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.973658] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.979682] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 68.987353] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 68.993381] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.002178] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.010980] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.020624] pcieport 0000:00:01.1: PME# disabled
[ 69.084588] pcieport 0000:02:02.0: PME# disabled
[ 69.101016] pcieport 0000:02:02.0: pci_restore_vc_state: i:1 pos:328
[ 69.107376] pcieport 0000:02:02.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 69.113998] pcieport 0000:02:02.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 69.120646] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.128225] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.135795] pcieport 0000:02:02.0: pci_vc_do_save_buffer: len:12
[ 69.141818] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.149407] pcieport 0000:02:02.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.149972] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 69.156981] pcieport 0000:02:02.0: pci_vc_do_save_buffer: len:12
[ 69.157058] pcieport 0000:02:02.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.157063] pcieport 0000:02:02.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.157067] pcieport 0000:02:02.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.157261] pcieport 0000:02:02.0: PME# disabled
[ 69.165517] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 69.171639] pci_bus 0000:0a: busn_res: [bus 0a] is released
[ 69.180336] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 69.188880] pcieport 0000:02:08.0: PME# disabled
[ 69.203089] pcieport 0000:02:08.0: pci_restore_vc_state: i:1 pos:328
[ 69.203091] pcieport 0000:02:08.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 69.203093] pcieport 0000:02:08.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 69.203097] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 69.203100] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.203101] pcieport 0000:02:08.0: pci_vc_do_save_buffer: len:76
[ 69.203108] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 69.211361] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 69.216693] pcieport 0000:02:08.0: pci_vc_save_restore_dwords: pos:376 dwords:16
[ 69.216758] pcieport 0000:02:08.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.216763] pcieport 0000:02:08.0: pci_vc_do_save_buffer: len:76
[ 69.216804] pcieport 0000:02:08.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.225139] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 69.229513] pcieport 0000:02:08.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.229520] pcieport 0000:02:08.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.229614] pcieport 0000:02:08.0: PME# disabled
[ 69.229829] pci_bus 0000:0b: busn_res: [bus 0b] is released
[ 69.236231] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 69.242730] pcieport 0000:02:09.0: PME# disabled
[ 69.249579] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 69.257048] pci_bus 0000:0c: dev 00, dec refcount to 0
[ 69.257060] pci_bus 0000:0c: dev 00, released physical slot 9
[ 69.257223] pci_bus 0000:0c: busn_res: [bus 0c] is released
[ 69.264928] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 69.270998] pcieport 0000:02:0a.0: PME# disabled
[ 69.278705] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 69.302025] pcieport 0000:02:0a.0: pci_restore_vc_state: i:1 pos:328
[ 69.307986] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 69.316417] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 69.316419] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 69.316428] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.316432] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.316433] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: len:12
[ 69.316446] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:0
[ 69.316453] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 69.316460] pcieport 0000:02:0a.0: pci_vc_do_save_buffer: len:12
[ 69.316514] pcieport 0000:02:0a.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 69.316521] pcieport 0000:02:0a.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 69.316525] pcieport 0000:02:0a.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 69.316660] pcieport 0000:02:0a.0: PME# disabled
[ 69.324439] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 69.333201] pci_bus 0000:0d: busn_res: [bus 0d] is released
[ 69.342639] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 69.347130] plx_dma 0000:01:00.1: PME# disabled
[ 69.352951] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 69.546955] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 69.554803] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 69.562563] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 69.570767] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 69.577487] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 69.584208] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 69.591884] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 69.597910] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 69.605583] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 69.611806] pcieport 0000:00:01.1: PME# enabled
[ 69.623179] plx_dma 0000:01:00.2: PME# disabled
[ 69.628424] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 69.640379] plx_dma 0000:01:00.3: PME# disabled
[ 69.656336] plx_dma 0000:01:00.4: PME# disabled
[ 72.886830] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 72.904973] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 72.911333] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 72.917963] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 72.924583] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 72.932152] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 72.938089] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 72.945665] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 72.951596] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 72.960292] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 72.968989] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 72.978623] pcieport 0000:00:01.1: PME# disabled
[ 73.099568] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 73.107774] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 73.115786] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 73.123878] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 73.131892] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 73.139559] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 73.147222] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 73.155220] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 73.162968] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 73.170892] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 73.178913] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 73.186579] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 73.194234] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 73.201890] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 73.209637] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 73.217296] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 73.225411] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 73.232025] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 73.238644] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.246217] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.252153] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.259736] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.265707] pcieport 0000:00:01.1: PME# enabled
[ 73.282253] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 73.289128] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 73.306977] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 73.313335] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 73.319962] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 73.326583] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.334159] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.340092] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.347671] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.353617] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 73.362320] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 73.371018] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 73.380445] pcieport 0000:00:01.1: PME# disabled
[ 73.430387] pcieport 0000:00:01.0: aer_inject: Injecting errors 00000000/00004000 into device 0000:00:01.0
[ 73.440086] pcieport 0000:00:01.0: AER: Uncorrected (Fatal) error received: 0000:00:01.0
[ 73.448186] pcieport 0000:00:01.0: PCIe Bus Error: severity=Uncorrected (Fatal), type=Transaction Layer, (Requester ID)
[ 73.458969] pcieport 0000:00:01.0: device [8086:1901] error status/mask=00004000/00000000
[ 73.467328] pcieport 0000:00:01.0: [14] CmpltTO
[ 73.473434] xdma_error_detected: >>>
[ 73.477012] xdma:xdma_error_detected: dev 0x00000000591acbe0,0x00000000c77a5002, frozen state error, reset controller
[ 73.487626] xdma:xdma_device_offline: pdev 0x00000000591acbe0, xdev 0x00000000947398f0.
[ 73.495738] xdma:xdma_device_offline: xdev 0x00000000947398f0, done.
[ 73.509969] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 73.518157] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 73.526165] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 73.534267] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 73.542269] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 73.549931] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 73.557591] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 73.565605] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 73.573359] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 73.581278] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 73.589286] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 73.596945] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 73.604609] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 73.612272] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 73.620024] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 73.627691] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 73.635804] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 73.642419] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 73.649037] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.656602] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.662525] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 73.670102] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 73.676118] pcieport 0000:00:01.1: PME# enabled
[ 73.692066] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 74.509998] pcieport 0000:00:01.0: AER: Root Port link has been reset (0)
[ 74.516820] pcieport 0000:01:00.0: pci_restore_vc_state: i:1 pos:328
[ 74.523169] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 74.529783] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 74.536405] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.544059] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.551630] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 74.557663] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.565327] pcieport 0000:01:00.0: pci_vc_save_restore_dwords: pos:376 dwords:16
[ 74.572741] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.580318] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 74.586354] pcieport 0000:01:00.0: restoring config space at offset 0x3c (was 0x100, writing 0x12010b)
[ 74.595676] pcieport 0000:01:00.0: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 74.604377] pcieport 0000:01:00.0: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 74.613076] pcieport 0000:01:00.0: restoring config space at offset 0x24 (was 0x1fff1, writing 0x91119001)
[ 74.622730] pcieport 0000:01:00.0: restoring config space at offset 0x20 (was 0xfff0, writing 0xdf10de00)
[ 74.632300] pcieport 0000:01:00.0: restoring config space at offset 0x1c (was 0x1f1, writing 0x8121)
[ 74.641427] pcieport 0000:01:00.0: restoring config space at offset 0x18 (was 0x0, writing 0xd0201)
[ 74.650479] pcieport 0000:01:00.0: restoring config space at offset 0x10 (was 0x0, writing 0xdf200000)
[ 74.659787] pcieport 0000:01:00.0: restoring config space at offset 0xc (was 0x810000, writing 0x810010)
[ 74.669275] pcieport 0000:01:00.0: restoring config space at offset 0x4 (was 0x100000, writing 0x100407)
[ 74.678805] pcieport 0000:01:00.0: saving config space at offset 0x0 (reading 0x872510b5)
[ 74.686984] pcieport 0000:01:00.0: saving config space at offset 0x4 (reading 0x100407)
[ 74.694996] pcieport 0000:01:00.0: saving config space at offset 0x8 (reading 0x60400ca)
[ 74.703094] pcieport 0000:01:00.0: saving config space at offset 0xc (reading 0x810010)
[ 74.711114] pcieport 0000:01:00.0: saving config space at offset 0x10 (reading 0xdf200000)
[ 74.719381] pcieport 0000:01:00.0: saving config space at offset 0x14 (reading 0x0)
[ 74.727031] pcieport 0000:01:00.0: saving config space at offset 0x18 (reading 0xd0201)
[ 74.735027] pcieport 0000:01:00.0: saving config space at offset 0x1c (reading 0x8121)
[ 74.742939] pcieport 0000:01:00.0: saving config space at offset 0x20 (reading 0xdf10de00)
[ 74.751201] pcieport 0000:01:00.0: saving config space at offset 0x24 (reading 0x91119001)
[ 74.759472] pcieport 0000:01:00.0: saving config space at offset 0x28 (reading 0x0)
[ 74.767134] pcieport 0000:01:00.0: saving config space at offset 0x2c (reading 0x0)
[ 74.774788] pcieport 0000:01:00.0: saving config space at offset 0x30 (reading 0x0)
[ 74.782441] pcieport 0000:01:00.0: saving config space at offset 0x34 (reading 0x40)
[ 74.790188] pcieport 0000:01:00.0: saving config space at offset 0x38 (reading 0x0)
[ 74.797844] pcieport 0000:01:00.0: saving config space at offset 0x3c (reading 0x12010b)
[ 74.806027] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 74.812641] pcieport 0000:01:00.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 74.819255] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.826911] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.834483] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 74.840505] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 74.848160] pcieport 0000:01:00.0: pci_vc_save_restore_dwords: pos:376 dwords:16
[ 74.855580] pcieport 0000:01:00.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 74.863150] pcieport 0000:01:00.0: pci_vc_do_save_buffer: len:76
[ 76.390998] sched: RT throttling activated
[ 78.616988] pcieport 0000:00:01.1: power state changed by ACPI to D0
[ 78.636106] pcieport 0000:00:01.1: pci_restore_vc_state: i:1 pos:256
[ 78.642483] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 78.649139] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 78.655790] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.663376] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 78.669325] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.676919] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 78.682969] pcieport 0000:00:01.1: restoring config space at offset 0x2c (was 0x0, writing 0x0)
[ 78.691694] pcieport 0000:00:01.1: restoring config space at offset 0x28 (was 0x0, writing 0x0)
[ 78.700420] pcieport 0000:00:01.1: restoring config space at offset 0x24 (was 0x1fff1, writing 0x1fff1)
[ 78.710200] pcieport 0000:00:01.1: PME# disabled
[ 78.838975] pcieport 0000:00:01.1: saving config space at offset 0x0 (reading 0x19058086)
[ 78.847182] pcieport 0000:00:01.1: saving config space at offset 0x4 (reading 0x100407)
[ 78.855215] pcieport 0000:00:01.1: saving config space at offset 0x8 (reading 0x6040005)
[ 78.863331] pcieport 0000:00:01.1: saving config space at offset 0xc (reading 0x810010)
[ 78.871363] pcieport 0000:00:01.1: saving config space at offset 0x10 (reading 0x0)
[ 78.879043] pcieport 0000:00:01.1: saving config space at offset 0x14 (reading 0x0)
[ 78.886728] pcieport 0000:00:01.1: saving config space at offset 0x18 (reading 0xe0e00)
[ 78.894762] pcieport 0000:00:01.1: saving config space at offset 0x1c (reading 0xf0)
[ 78.902529] pcieport 0000:00:01.1: saving config space at offset 0x20 (reading 0xfff0)
[ 78.910469] pcieport 0000:00:01.1: saving config space at offset 0x24 (reading 0x1fff1)
[ 78.918498] pcieport 0000:00:01.1: saving config space at offset 0x28 (reading 0x0)
[ 78.926176] pcieport 0000:00:01.1: saving config space at offset 0x2c (reading 0x0)
[ 78.933857] pcieport 0000:00:01.1: saving config space at offset 0x30 (reading 0x0)
[ 78.941537] pcieport 0000:00:01.1: saving config space at offset 0x34 (reading 0x88)
[ 78.949299] pcieport 0000:00:01.1: saving config space at offset 0x38 (reading 0x0)
[ 78.956985] pcieport 0000:00:01.1: saving config space at offset 0x3c (reading 0x12010b)
[ 78.965256] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:1 pos:256
[ 78.971890] pcieport 0000:00:01.1: pci_vc_do_save_buffer: buf:0 pos:256
[ 78.978537] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.986126] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 78.992093] pcieport 0000:00:01.1: pci_vc_do_save_buffer: i:0 evcc:0 parb_offset:0
[ 78.999683] pcieport 0000:00:01.1: pci_vc_do_save_buffer: len:8
[ 79.005664] pcieport 0000:00:01.1: PME# enabled
[ 79.023036] pcieport 0000:00:01.1: power state changed by ACPI to D3cold
[ 101.457576] watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [irq/122-aerdrv:129]
[ 101.465153] Modules linked in: xdma(O) aer_inject xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6tai
[ 101.465206] drm_kms_helper syscopyarea sysfillrect igb sysimgblt ptp fb_sys_fops libahci crc32c_intel pps_core serio_raw dca i2c_algo_bit libata drm video
[ 101.566072] CPU: 2 PID: 129 Comm: irq/122-aerdrv Tainted: G S O 5.10.0-1.keithv3.el7.x86_64 #1
[ 101.575814] Hardware name: AMI AM G6x/msd/AM G6x/msd, BIOS 4.08.01 02/19/2019
[ 101.582950] RIP: 0010:pci_mmcfg_read+0xb7/0xe0
[ 101.587403] Code: 63 c5 48 01 d8 66 8b 00 0f b7 c0 41 89 04 24 eb dc 49 63 c5 48 01 d8 8a 00 0f b6 c0 41 89 04 24 eb cb 49 63 c5 48 01 d8 8b 00 <41> 89 04 24 eb bd e8 be f4 78 ff 5b 41 c7 04 24 ff ff ff ff b8 ea
[ 101.606158] RSP: 0018:ffffc90000303b68 EFLAGS: 00000286
[ 101.611391] RAX: 00000000b701000d RBX: ffffc90010208000 RCX: 0000000000000f24
[ 101.618531] RDX: 00000000000000ff RSI: 0000000000000002 RDI: 0000000000000000
[ 101.625673] RBP: ffffc90000303b90 R08: 0000000000000004 R09: ffffc90000303bc4
[ 101.632811] R10: 0000000000000008 R11: ffffc900003038e0 R12: ffffc90000303bc4
[ 101.639944] R13: 0000000000000f24 R14: 0000000000008000 R15: 0000000000000004
[ 101.647082] FS: 0000000000000000(0000) GS:ffff88845dc80000(0000) knlGS:0000000000000000
[ 101.655174] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 101.660920] CR2: 00007f9369637b80 CR3: 000000000240a005 CR4: 00000000003706e0
[ 101.668057] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 101.675190] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 101.682323] Call Trace:
[ 101.684780] raw_pci_read+0x35/0x40
[ 101.688277] pci_read+0x2c/0x30
[ 101.691428] pci_bus_read_config_dword+0x4a/0x70
[ 101.696051] pci_read_config_dword+0x23/0x40
[ 101.700331] pci_find_next_ext_capability.part.24+0x6b/0xe0
[ 101.705912] pci_find_ext_capability.part.25+0x12/0x20
[ 101.711058] pci_find_ext_capability+0x1a/0x20
[ 101.715509] pci_restore_vc_state+0x32/0xb0
[ 101.719695] ? pcie_capability_write_word+0x32/0x90
[ 101.724579] pci_restore_state.part.44+0x177/0x420
[ 101.729377] pci_restore_state+0x18/0x20
[ 101.733310] pcie_portdrv_slot_reset+0x12/0x30
[ 101.737765] report_slot_reset+0x46/0x80
[ 101.741695] ? merge_result.part.4+0x40/0x40
[ 101.745971] pci_walk_bus+0x48/0x90
[ 101.749466] pci_walk_bridge+0x1d/0x30
[ 101.753223] pcie_do_recovery+0x1bd/0x270
[ 101.757241] ? aer_dev_correctable_show+0xd0/0xd0
[ 101.761948] ? aer_dev_correctable_show+0xd0/0xd0
[ 101.766657] aer_process_err_devices+0xd0/0xe9
[ 101.771104] aer_isr.cold.25+0x54/0xa3
[ 101.774862] ? irq_finalize_oneshot.part.48+0xf0/0xf0
[ 101.779924] irq_thread_fn+0x24/0x60
[ 101.783516] irq_thread+0xea/0x170
[ 101.786926] ? irq_forced_thread_fn+0x80/0x80
[ 101.791292] ? irq_thread_check_affinity+0xf0/0xf0
[ 101.796088] kthread+0x126/0x140
[ 101.799330] ? kthread_park+0x90/0x90
[ 101.803004] ret_from_fork+0x22/0x30
[ 111.539063] pcieport 0000:02:01.0: pci_restore_vc_state: i:1 pos:328
[ 111.545503] pcieport 0000:02:01.0: pci_vc_do_save_buffer: buf:1 pos:328
[ 129.266656] pcieport 0000:02:01.0: pci_vc_do_save_buffer: buf:0 pos:328
[ 134.720858] pcieport 0000:02:01.0: pci_vc_do_save_buffer: i:0 evcc:1 parb_offset:48
[ 134.728712] pcieport 0000:02:01.0: pci_vc_do_save_buffer: i:1 evcc:1 parb_offset:0
[ 136.397775] rcu: INFO: rcu_sched self-detected stall on CPU
[ 136.410927] rcu: 2-....: (1 GPs behind) idle=7b2/1/0x4000000000000000 softirq=5916/5917 fqs=14413
[ 136.419972] (t=60000 jiffies g=12137 q=11828)
[ 136.424422] NMI backtrace for cpu 2
[ 136.427915] CPU: 2 PID: 129 Comm: irq/122-aerdrv Tainted: G S O L 5.10.0-1.keithv3.el7.x86_64 #1
[ 136.437651] Hardware name: AMI AM G6x/msd/AM G6x/msd, BIOS 4.08.01 02/19/2019
[ 136.444785] Call Trace:
[ 136.447237] <IRQ>
[ 136.449263] dump_stack+0x74/0x92
[ 136.452595] nmi_cpu_backtrace.cold.8+0x32/0x68
[ 136.457134] ? lapic_can_unplug_cpu+0x80/0x80
[ 136.461494] nmi_trigger_cpumask_backtrace+0xe6/0xf0
[ 136.466467] arch_trigger_cpumask_backtrace+0x19/0x20
[ 136.471526] rcu_dump_cpu_stacks+0xb6/0xe6
[ 136.475635] rcu_sched_clock_irq.cold.92+0x124/0x3a9
[ 136.480610] ? account_system_index_time+0x94/0xa0
[ 136.485417] ? tick_sched_do_timer+0x60/0x60
[ 136.489696] update_process_times+0x93/0xd0
[ 136.493892] tick_sched_handle.isra.23+0x25/0x60
[ 136.498517] tick_sched_timer+0x6b/0x80
[ 136.502360] __hrtimer_run_queues+0x105/0x240
[ 136.506726] hrtimer_interrupt+0x101/0x220
[ 136.510836] __sysvec_apic_timer_interrupt+0x62/0xe0
[ 136.515815] asm_call_irq_on_stack+0x12/0x20
[ 136.520092] </IRQ>
[ 136.522202] sysvec_apic_timer_interrupt+0x7d/0x90
[ 136.527003] asm_sysvec_apic_timer_interrupt+0x12/0x20
[ 136.532152] RIP: 0010:vprintk_emit+0x10b/0x250
[ 136.536601] Code: 1f 44 00 00 e8 e6 0b 00 00 8b 7d d4 48 8b 55 c8 89 de 4d 89 f8 4c 89 f1 e8 62 fa ff ff 89 c3 e8 eb 0b 00 00 48 8b 7d c0 57 9d <0f> 1f 44 00 00 45 84 e4 74 1f 80 3d 95 0a 6f 01 00 74 05 e8 3d d6
[ 136.555356] RSP: 0018:ffffc90000303968 EFLAGS: 00000246
[ 136.560583] RAX: 0000000000000046 RBX: 0000000000000046 RCX: 0000000000001d91
[ 136.567718] RDX: ffffffff825d9628 RSI: 80000000ffffc9db RDI: 0000000000000246
[ 136.574852] RBP: ffffc900003039b0 R08: ffffffff8246a940 R09: 0000000000000000
[ 136.581992] R10: ffffffff82b21518 R11: ffffffff82b21516 R12: 0000000000000000
[ 136.589126] R13: ffffffff82176221 R14: ffffffff8215574f R15: ffffc90000303a60
[ 136.596267] dev_vprintk_emit+0x13b/0x15f
[ 136.600286] dev_printk_emit+0x4e/0x65
[ 136.604048] ? pci_mmcfg_read+0xb7/0xe0
[ 136.607890] __dev_printk+0x58/0x6b
[ 136.611383] _dev_warn+0x6c/0x83
[ 136.614622] ? pci_bus_read_config_dword+0x4a/0x70
[ 136.619418] pci_vc_do_save_buffer+0x251/0x5ab
[ 136.623869] pci_vc_do_save_buffer+0x87/0x5ab
[ 136.628238] ? _dev_warn+0x6c/0x83
[ 136.631642] pci_restore_vc_state+0x7e/0xb0
[ 136.635839] pci_restore_state.part.44+0x177/0x420
[ 136.640636] pci_restore_state+0x18/0x20
[ 136.644569] pcie_portdrv_slot_reset+0x12/0x30
[ 136.649017] report_slot_reset+0x46/0x80
[ 136.652945] ? merge_result.part.4+0x40/0x40
[ 136.657221] pci_walk_bus+0x48/0x90
[ 136.660715] pci_walk_bridge+0x1d/0x30
[ 136.664471] pcie_do_recovery+0x1bd/0x270
[ 136.668483] ? aer_dev_correctable_show+0xd0/0xd0
[ 136.673192] ? aer_dev_correctable_show+0xd0/0xd0
[ 136.677901] aer_process_err_devices+0xd0/0xe9
[ 136.682351] aer_isr.cold.25+0x54/0xa3
[ 136.686108] ? irq_finalize_oneshot.part.48+0xf0/0xf0
[ 136.691167] irq_thread_fn+0x24/0x60
[ 136.694754] irq_thread+0xea/0x170
[ 136.698169] ? irq_forced_thread_fn+0x80/0x80
[ 136.702533] ? irq_thread_check_affinity+0xf0/0xf0
[ 136.707330] kthread+0x126/0x140
[ 136.710572] ? kthread_park+0x90/0x90
[ 136.714247] ret_from_fork+0x22/0x30
[ 136.717927] pcieport 0000:02:01.0: pci_vc_do_save_buffer: len:76
[ 161.459414] watchdog: BUG: soft lockup - CPU#2 stuck for 22s! [irq/122-aerdrv:129]
[ 161.459417] Modules linked in: xdma(O) aer_inject xt_CHECKSUM iptable_mangle xt_MASQUERADE iptable_nat nf_nat xt_conntrack nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ipt_REJECT nf_reject_ipv4 tun ebtable_filter ebtables ip6tai
[ 161.459460] drm_kms_helper syscopyarea sysfillrect igb sysimgblt ptp fb_sys_fops libahci crc32c_intel pps_core serio_raw dca i2c_algo_bit libata drm video
[ 161.459468] CPU: 2 PID: 129 Comm: irq/122-aerdrv Tainted: G S O L 5.10.0-1.keithv3.el7.x86_64 #1
[ 161.459470] Hardware name: AMI AM G6x/msd/AM G6x/msd, BIOS 4.08.01 02/19/2019
[ 161.459471] RIP: 0010:console_unlock+0x374/0x510
[ 161.459475] Code: ff e8 10 f9 ff ff 85 c0 0f 85 4f ff ff ff e8 c3 f8 ff ff 85 c0 0f 85 18 fd ff ff e9 3d ff ff ff e8 71 2b 00 00 4c 89 ff 57 9d <0f> 1f 44 00 00 8b 85 44 ff ff ff 85 c0 0f 84 3a fd ff ff e8 74 03
[ 161.459476] RSP: 0018:ffffc90000303898 EFLAGS: 00000247
[ 161.459478] RAX: 0000000000000000 RBX: ffffffff82af03ac RCX: ffffffff815dba40
[ 161.459479] RDX: 0000000000000000 RSI: ffffffff82af2420 RDI: 0000000000000247
[ 161.459480] RBP: ffffc90000303958 R08: 0000019b07f91685 R09: 0000000000000002
[ 161.459481] R10: 32303a3030303020 R11: 74726f7065696370 R12: 0000000000000000
[ 161.459482] R13: 0000000000000000 R14: ffffffff827ecfa0 R15: 0000000000000247
[ 161.459483] FS: 0000000000000000(0000) GS:ffff88845dc80000(0000) knlGS:0000000000000000
[ 161.459485] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 161.459486] CR2: 00007f9369637b80 CR3: 000000000240a005 CR4: 00000000003706e0
[ 161.459487] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 161.459488] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 161.459489] Call Trace:
[ 161.459491] vprintk_emit+0x14b/0x250
[ 161.459493] dev_vprintk_emit+0x13b/0x15f
[ 161.459497] dev_printk_emit+0x4e/0x65
[ 161.459499] ? pci_mmcfg_read+0xb7/0xe0
[ 161.459503] __dev_printk+0x58/0x6b
[ 161.459505] _dev_warn+0x6c/0x83
[ 161.459508] ? pci_bus_read_config_dword+0x4a/0x70
[ 161.459510] pci_vc_do_save_buffer+0x576/0x5ab
...
