Sure, also attached.
cephuser@cephnode6:~$ ethtool -S enp3s0f0
NIC statistics:
rx_packets: 3103528
tx_packets: 20954382
rx_bytes: 1385006975
tx_bytes: 30063866207
rx_broadcast: 8
tx_broadcast: 2
rx_multicast: 14098
tx_multicast: 476
multicast: 14098
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 1563318
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 1385006975
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 292684
tx_queue_0_bytes: 216244150
tx_queue_0_restart: 0
tx_queue_1_packets: 6489256
tx_queue_1_bytes: 9529075383
tx_queue_1_restart: 0
tx_queue_2_packets: 325263
tx_queue_2_bytes: 330734519
tx_queue_2_restart: 0
tx_queue_3_packets: 3074167
tx_queue_3_bytes: 4363429551
tx_queue_3_restart: 0
tx_queue_4_packets: 319961
tx_queue_4_bytes: 242633539
tx_queue_4_restart: 0
tx_queue_5_packets: 252717
tx_queue_5_bytes: 191717682
tx_queue_5_restart: 0
tx_queue_6_packets: 3402851
tx_queue_6_bytes: 4966526009
tx_queue_6_restart: 0
tx_queue_7_packets: 6797483
tx_queue_7_bytes: 10139687288
tx_queue_7_restart: 0
rx_queue_0_packets: 215189
rx_queue_0_bytes: 153838496
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 528614
rx_queue_1_bytes: 192679289
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 186822
rx_queue_2_bytes: 141708803
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 173099
rx_queue_3_bytes: 131321568
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 147423
rx_queue_4_bytes: 111807376
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 133384
rx_queue_5_bytes: 110116165
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 1598989
rx_queue_6_bytes: 440079884
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 120008
rx_queue_7_bytes: 91041282
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp3s0f1
NIC statistics:
rx_packets: 2417818
tx_packets: 1605247
rx_bytes: 1790337041
tx_bytes: 1268847302
rx_broadcast: 80
tx_broadcast: 11
rx_multicast: 14235
tx_multicast: 463
multicast: 14235
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 511041
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 1790337041
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 187486
tx_queue_0_bytes: 158004014
tx_queue_0_restart: 0
tx_queue_1_packets: 292695
tx_queue_1_bytes: 216338071
tx_queue_1_restart: 0
tx_queue_2_packets: 230556
tx_queue_2_bytes: 187701434
tx_queue_2_restart: 0
tx_queue_3_packets: 175072
tx_queue_3_bytes: 132739732
tx_queue_3_restart: 0
tx_queue_4_packets: 201110
tx_queue_4_bytes: 169995905
tx_queue_4_restart: 0
tx_queue_5_packets: 164302
tx_queue_5_bytes: 122529296
tx_queue_5_restart: 0
tx_queue_6_packets: 176127
tx_queue_6_bytes: 142367535
tx_queue_6_restart: 0
tx_queue_7_packets: 177899
tx_queue_7_bytes: 132745095
tx_queue_7_restart: 0
rx_queue_0_packets: 244981
rx_queue_0_bytes: 175468464
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 226753
rx_queue_1_bytes: 164297931
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 309263
rx_queue_2_bytes: 224912817
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 344819
rx_queue_3_bytes: 256411809
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 413956
rx_queue_4_bytes: 306366538
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 222891
rx_queue_5_bytes: 163512696
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 309235
rx_queue_6_bytes: 234497920
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 345920
rx_queue_7_bytes: 255197594
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp4s0f0
NIC statistics:
rx_packets: 263095022
tx_packets: 165984477
rx_bytes: 358188428243
tx_bytes: 224059860328
rx_broadcast: 0
tx_broadcast: 0
rx_multicast: 14260
tx_multicast: 475
multicast: 14260
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 10618678
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 358188428243
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 24523878
tx_queue_0_bytes: 35555825370
tx_queue_0_restart: 0
tx_queue_1_packets: 22792805
tx_queue_1_bytes: 33160725172
tx_queue_1_restart: 0
tx_queue_2_packets: 32186938
tx_queue_2_bytes: 42938373893
tx_queue_2_restart: 0
tx_queue_3_packets: 27532192
tx_queue_3_bytes: 35219225817
tx_queue_3_restart: 0
tx_queue_4_packets: 12999613
tx_queue_4_bytes: 16197252415
tx_queue_4_restart: 0
tx_queue_5_packets: 20425057
tx_queue_5_bytes: 25328350119
tx_queue_5_restart: 0
tx_queue_6_packets: 13115439
tx_queue_6_bytes: 17344340207
tx_queue_6_restart: 0
tx_queue_7_packets: 12408555
tx_queue_7_bytes: 17651829427
tx_queue_7_restart: 0
rx_queue_0_packets: 28425656
rx_queue_0_bytes: 37636565075
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 8984649
rx_queue_1_bytes: 2071222507
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 36295986
rx_queue_2_bytes: 54200457391
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 23349464
rx_queue_3_bytes: 33468571056
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 5805069
rx_queue_4_bytes: 715544992
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 47433585
rx_queue_5_bytes: 67747439829
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 40708301
rx_queue_6_bytes: 53328721761
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 72092312
rx_queue_7_bytes: 107967525544
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp4s0f1
NIC statistics:
rx_packets: 100608455
tx_packets: 166526987
rx_bytes: 140666343603
tx_bytes: 222986388680
rx_broadcast: 91
tx_broadcast: 8
rx_multicast: 14286
tx_multicast: 469
multicast: 14286
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 9252151
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 140666343603
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 9639934
tx_queue_0_bytes: 13277298005
tx_queue_0_restart: 0
tx_queue_1_packets: 21458007
tx_queue_1_bytes: 28789498862
tx_queue_1_restart: 0
tx_queue_2_packets: 22103147
tx_queue_2_bytes: 28134369759
tx_queue_2_restart: 0
tx_queue_3_packets: 26218962
tx_queue_3_bytes: 32565488967
tx_queue_3_restart: 0
tx_queue_4_packets: 26578778
tx_queue_4_bytes: 39751527579
tx_queue_4_restart: 0
tx_queue_5_packets: 24551167
tx_queue_5_bytes: 36050216125
tx_queue_5_restart: 0
tx_queue_6_packets: 18808095
tx_queue_6_bytes: 27305113278
tx_queue_6_restart: 0
tx_queue_7_packets: 17168897
tx_queue_7_bytes: 16446768013
tx_queue_7_restart: 0
rx_queue_0_packets: 25074819
rx_queue_0_bytes: 37499171327
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 2573243
rx_queue_1_bytes: 780414215
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 35512962
rx_queue_2_bytes: 53302135608
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 3976283
rx_queue_3_bytes: 679395976
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 24670023
rx_queue_4_bytes: 36883071881
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 3467062
rx_queue_5_bytes: 4846515985
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 4683116
rx_queue_6_bytes: 5428970649
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 650947
rx_queue_7_bytes: 844234142
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp3s0f0
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:03:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp3s0f1
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:03:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp4s0f0
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:04:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp4s0f1
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:04:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ dmesg -TL | grep igb
[Thu Jul 18 10:36:09 2019] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.4.0-k
[Thu Jul 18 10:36:09 2019] igb: Copyright (c) 2007-2014 Intel Corporation.
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: added PHC on eth0
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: eth0: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:20
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: eth0: PBA No: Unknown
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: added PHC on eth1
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: eth1: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:21
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: eth1: PBA No: Unknown
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: added PHC on eth2
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: eth2: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:22
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: eth2: PBA No: Unknown
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: added PHC on eth3
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: eth3: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:23
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: eth3: PBA No: Unknown
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1 enp4s0f1: renamed from eth3
[Thu Jul 18 10:36:11 2019] igb 0000:03:00.0 enp3s0f0: renamed from eth0
[Thu Jul 18 10:36:11 2019] igb 0000:03:00.1 enp3s0f1: renamed from eth1
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0 enp4s0f0: renamed from eth2
[Thu Jul 18 10:36:18 2019] igb 0000:04:00.1 enp4s0f1: igb: enp4s0f1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:18 2019] igb 0000:03:00.0 enp3s0f0: igb: enp3s0f0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:19 2019] igb 0000:04:00.0 enp4s0f0: igb: enp4s0f0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:19 2019] igb 0000:03:00.1 enp3s0f1: igb: enp3s0f1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
cephuser@cephnode6:~$
cephuser@cephnode6:~$ uptime
14:39:44 up 4:03, 1 user, load average: 1.84, 1.59, 1.57
cephuser@cephnode6:~$
NIC statistics:
rx_packets: 3103528
tx_packets: 20954382
rx_bytes: 1385006975
tx_bytes: 30063866207
rx_broadcast: 8
tx_broadcast: 2
rx_multicast: 14098
tx_multicast: 476
multicast: 14098
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 1563318
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 1385006975
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 292684
tx_queue_0_bytes: 216244150
tx_queue_0_restart: 0
tx_queue_1_packets: 6489256
tx_queue_1_bytes: 9529075383
tx_queue_1_restart: 0
tx_queue_2_packets: 325263
tx_queue_2_bytes: 330734519
tx_queue_2_restart: 0
tx_queue_3_packets: 3074167
tx_queue_3_bytes: 4363429551
tx_queue_3_restart: 0
tx_queue_4_packets: 319961
tx_queue_4_bytes: 242633539
tx_queue_4_restart: 0
tx_queue_5_packets: 252717
tx_queue_5_bytes: 191717682
tx_queue_5_restart: 0
tx_queue_6_packets: 3402851
tx_queue_6_bytes: 4966526009
tx_queue_6_restart: 0
tx_queue_7_packets: 6797483
tx_queue_7_bytes: 10139687288
tx_queue_7_restart: 0
rx_queue_0_packets: 215189
rx_queue_0_bytes: 153838496
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 528614
rx_queue_1_bytes: 192679289
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 186822
rx_queue_2_bytes: 141708803
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 173099
rx_queue_3_bytes: 131321568
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 147423
rx_queue_4_bytes: 111807376
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 133384
rx_queue_5_bytes: 110116165
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 1598989
rx_queue_6_bytes: 440079884
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 120008
rx_queue_7_bytes: 91041282
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp3s0f1
NIC statistics:
rx_packets: 2417818
tx_packets: 1605247
rx_bytes: 1790337041
tx_bytes: 1268847302
rx_broadcast: 80
tx_broadcast: 11
rx_multicast: 14235
tx_multicast: 463
multicast: 14235
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 511041
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 1790337041
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 187486
tx_queue_0_bytes: 158004014
tx_queue_0_restart: 0
tx_queue_1_packets: 292695
tx_queue_1_bytes: 216338071
tx_queue_1_restart: 0
tx_queue_2_packets: 230556
tx_queue_2_bytes: 187701434
tx_queue_2_restart: 0
tx_queue_3_packets: 175072
tx_queue_3_bytes: 132739732
tx_queue_3_restart: 0
tx_queue_4_packets: 201110
tx_queue_4_bytes: 169995905
tx_queue_4_restart: 0
tx_queue_5_packets: 164302
tx_queue_5_bytes: 122529296
tx_queue_5_restart: 0
tx_queue_6_packets: 176127
tx_queue_6_bytes: 142367535
tx_queue_6_restart: 0
tx_queue_7_packets: 177899
tx_queue_7_bytes: 132745095
tx_queue_7_restart: 0
rx_queue_0_packets: 244981
rx_queue_0_bytes: 175468464
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 226753
rx_queue_1_bytes: 164297931
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 309263
rx_queue_2_bytes: 224912817
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 344819
rx_queue_3_bytes: 256411809
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 413956
rx_queue_4_bytes: 306366538
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 222891
rx_queue_5_bytes: 163512696
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 309235
rx_queue_6_bytes: 234497920
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 345920
rx_queue_7_bytes: 255197594
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp4s0f0
NIC statistics:
rx_packets: 263095022
tx_packets: 165984477
rx_bytes: 358188428243
tx_bytes: 224059860328
rx_broadcast: 0
tx_broadcast: 0
rx_multicast: 14260
tx_multicast: 475
multicast: 14260
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 10618678
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 358188428243
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 24523878
tx_queue_0_bytes: 35555825370
tx_queue_0_restart: 0
tx_queue_1_packets: 22792805
tx_queue_1_bytes: 33160725172
tx_queue_1_restart: 0
tx_queue_2_packets: 32186938
tx_queue_2_bytes: 42938373893
tx_queue_2_restart: 0
tx_queue_3_packets: 27532192
tx_queue_3_bytes: 35219225817
tx_queue_3_restart: 0
tx_queue_4_packets: 12999613
tx_queue_4_bytes: 16197252415
tx_queue_4_restart: 0
tx_queue_5_packets: 20425057
tx_queue_5_bytes: 25328350119
tx_queue_5_restart: 0
tx_queue_6_packets: 13115439
tx_queue_6_bytes: 17344340207
tx_queue_6_restart: 0
tx_queue_7_packets: 12408555
tx_queue_7_bytes: 17651829427
tx_queue_7_restart: 0
rx_queue_0_packets: 28425656
rx_queue_0_bytes: 37636565075
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 8984649
rx_queue_1_bytes: 2071222507
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 36295986
rx_queue_2_bytes: 54200457391
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 23349464
rx_queue_3_bytes: 33468571056
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 5805069
rx_queue_4_bytes: 715544992
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 47433585
rx_queue_5_bytes: 67747439829
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 40708301
rx_queue_6_bytes: 53328721761
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 72092312
rx_queue_7_bytes: 107967525544
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp4s0f1
NIC statistics:
rx_packets: 100608455
tx_packets: 166526987
rx_bytes: 140666343603
tx_bytes: 222986388680
rx_broadcast: 91
tx_broadcast: 8
rx_multicast: 14286
tx_multicast: 469
multicast: 14286
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 9252151
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 140666343603
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 9639934
tx_queue_0_bytes: 13277298005
tx_queue_0_restart: 0
tx_queue_1_packets: 21458007
tx_queue_1_bytes: 28789498862
tx_queue_1_restart: 0
tx_queue_2_packets: 22103147
tx_queue_2_bytes: 28134369759
tx_queue_2_restart: 0
tx_queue_3_packets: 26218962
tx_queue_3_bytes: 32565488967
tx_queue_3_restart: 0
tx_queue_4_packets: 26578778
tx_queue_4_bytes: 39751527579
tx_queue_4_restart: 0
tx_queue_5_packets: 24551167
tx_queue_5_bytes: 36050216125
tx_queue_5_restart: 0
tx_queue_6_packets: 18808095
tx_queue_6_bytes: 27305113278
tx_queue_6_restart: 0
tx_queue_7_packets: 17168897
tx_queue_7_bytes: 16446768013
tx_queue_7_restart: 0
rx_queue_0_packets: 25074819
rx_queue_0_bytes: 37499171327
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 2573243
rx_queue_1_bytes: 780414215
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 35512962
rx_queue_2_bytes: 53302135608
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 3976283
rx_queue_3_bytes: 679395976
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 24670023
rx_queue_4_bytes: 36883071881
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 3467062
rx_queue_5_bytes: 4846515985
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 4683116
rx_queue_6_bytes: 5428970649
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 650947
rx_queue_7_bytes: 844234142
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp3s0f0
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:03:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp3s0f1
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:03:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp4s0f0
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:04:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp4s0f1
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:04:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ dmesg -TL | grep igb
[Thu Jul 18 10:36:09 2019] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.4.0-k
[Thu Jul 18 10:36:09 2019] igb: Copyright (c) 2007-2014 Intel Corporation.
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: added PHC on eth0
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: eth0: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:20
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: eth0: PBA No: Unknown
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: added PHC on eth1
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: eth1: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:21
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: eth1: PBA No: Unknown
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: added PHC on eth2
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: eth2: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:22
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: eth2: PBA No: Unknown
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: added PHC on eth3
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: eth3: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:23
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: eth3: PBA No: Unknown
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1 enp4s0f1: renamed from eth3
[Thu Jul 18 10:36:11 2019] igb 0000:03:00.0 enp3s0f0: renamed from eth0
[Thu Jul 18 10:36:11 2019] igb 0000:03:00.1 enp3s0f1: renamed from eth1
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0 enp4s0f0: renamed from eth2
[Thu Jul 18 10:36:18 2019] igb 0000:04:00.1 enp4s0f1: igb: enp4s0f1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:18 2019] igb 0000:03:00.0 enp3s0f0: igb: enp3s0f0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:19 2019] igb 0000:04:00.0 enp4s0f0: igb: enp4s0f0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:19 2019] igb 0000:03:00.1 enp3s0f1: igb: enp3s0f1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
cephuser@cephnode6:~$
cephuser@cephnode6:~$ uptime
14:39:44 up 4:03, 1 user, load average: 1.84, 1.59, 1.57
cephuser@cephnode6:~$
Kind regards
Geoffrey Rhodes
Geoffrey Rhodes
On Thu, 18 Jul 2019 at 14:02, Konstantin Shalygin <k0ste@xxxxxxxx> wrote:
I've been having an issue since upgrading my cluster to Mimic 6 months ago (previously installed with Luminous 12.2.1). All nodes that have the same PCIe network card seem to loose network connectivity randomly. (frequency ranges from a few days to weeks per host node) The affected nodes only have the Intel 82576 LAN Card in common, different motherboards, installed drives, RAM and even PSUs. Nodes that have the Intel I350 cards are not affected by the Mimic upgrade. Each host node has recommended RAM installed and has between 4 and 6 OSDs / sata hard drives installed. The cluster operated for over a year (Luminous) without a single issue, only after the Mimic upgrade did the issues begin with these nodes. The cluster is only used for CephFS (file storage, low intensity usage) and makes use of erasure data pool (K=4, M=2). I've tested many things, different kernel versions, different Ubuntu LTS releases, re-installation and even CENTOS 7, different releases of Mimic, different igb drivers. If I stop the ceph-osd daemons the issue does not occur. If I swap out the Intel 82576 card with the Intel I350 the issue is resolved. I haven't any more ideas other than replacing the cards but I feel the issue is linked to the ceph-osd daemon and a change in the Mimic release. Below are the various software versions and drivers I've tried and a log extract from a node that lost network connectivity. - Any help or suggestions would be greatly appreciated. *OS:* Ubuntu 16.04 / 18.04 and recently CENTOS 7 *Ceph Version:* Mimic (currently 13.2.6) *Network card:* 4-PORT 1GB INTEL 82576 LAN CARD (AOC-SG-I4) *Driver: * igb *Driver Versions:* 5.3.0-k / 5.3.5.22s / 5.4.0-k *Network Config:* 2 x bonded (LACP) 1GB nic for public net, 2 x bonded (LACP) 1GB nic for private net *Log errors:* Jun 27 12:10:28 cephnode5 kernel: [497346.638608] igb 0000:03:00.0 enp3s0f0: PCIe link lost, device now detached Jun 27 12:10:28 cephnode5 kernel: [497346.686752] igb 0000:04:00.1 enp4s0f1: PCIe link lost, device now detached Jun 27 12:10:29 cephnode5 kernel: [497347.550473] igb 0000:03:00.1 enp3s0f1: PCIe link lost, device now detached Jun 27 12:10:29 cephnode5 kernel: [497347.646785] igb 0000:04:00.0 enp4s0f0: PCIe link lost, device now detached Jun 27 12:10:43 cephnode5 ceph-osd[2575]: 2019-06-27 12:10:43.793 7f73ca637700 -1 osd.15 28497 heartbeat_check: no reply from 10.100.4.1:6809 osd.16 since back 2019-06 -27 12:10:27.438961 front 2019-06-27 12:10:23.338012 (cutoff 2019-06-27 12:10:23.796726) Jun 27 12:10:43 cephnode5 ceph-osd[2575]: 2019-06-27 12:10:43.793 7f73ca637700 -1 osd.15 28497 heartbeat_check: no reply from 10.100.6.1:6804 osd.20 since back 2019-06 -27 12:10:27.438961 front 2019-06-27 12:10:23.338012 (cutoff 2019-06-27 12:10:23.796726) Jun 27 12:10:43 cephnode5 ceph-osd[2575]: 2019-06-27 12:10:43.793 7f73ca637700 -1 osd.15 28497 heartbeat_check: no reply from 10.100.7.1:6803 osd.25 since back 2019-06 -27 12:10:23.338012 front 2019-06-27 12:10:23.338012 (cutoff 2019-06-27 12:10:23.796726) Jun 27 12:10:43 cephnode5 ceph-osd[2575]: 2019-06-27 12:10:43.793 7f73ca637700 -1 osd.15 28497 heartbeat_check: no reply from 10.100.8.1:6803 osd.30 since back 2019-06 -27 12:10:27.438961 front 2019-06-27 12:10:23.338012 (cutoff 2019-06-27 12:10:23.796726) Jun 27 12:10:43 cephnode5 ceph-osd[2575]: 2019-06-27 12:10:43.793 7f73ca637700 -1 osd.15 28497 heartbeat_check: no reply from 10.100.9.1:6808 osd.43 since back 2019-06 -27 12:10:23.338012 front 2019-06-27 12:10:23.338012 (cutoff 2019-06-27 12:10:23.796726)Paste your `ethtool -S <interface>`, `ethtool -i <interface>` and `dmesg -TL | grep igb`.
k
cephuser@cephnode6:~$ ethtool -S enp3s0f0
NIC statistics:
rx_packets: 3103528
tx_packets: 20954382
rx_bytes: 1385006975
tx_bytes: 30063866207
rx_broadcast: 8
tx_broadcast: 2
rx_multicast: 14098
tx_multicast: 476
multicast: 14098
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 1563318
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 1385006975
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 292684
tx_queue_0_bytes: 216244150
tx_queue_0_restart: 0
tx_queue_1_packets: 6489256
tx_queue_1_bytes: 9529075383
tx_queue_1_restart: 0
tx_queue_2_packets: 325263
tx_queue_2_bytes: 330734519
tx_queue_2_restart: 0
tx_queue_3_packets: 3074167
tx_queue_3_bytes: 4363429551
tx_queue_3_restart: 0
tx_queue_4_packets: 319961
tx_queue_4_bytes: 242633539
tx_queue_4_restart: 0
tx_queue_5_packets: 252717
tx_queue_5_bytes: 191717682
tx_queue_5_restart: 0
tx_queue_6_packets: 3402851
tx_queue_6_bytes: 4966526009
tx_queue_6_restart: 0
tx_queue_7_packets: 6797483
tx_queue_7_bytes: 10139687288
tx_queue_7_restart: 0
rx_queue_0_packets: 215189
rx_queue_0_bytes: 153838496
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 528614
rx_queue_1_bytes: 192679289
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 186822
rx_queue_2_bytes: 141708803
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 173099
rx_queue_3_bytes: 131321568
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 147423
rx_queue_4_bytes: 111807376
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 133384
rx_queue_5_bytes: 110116165
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 1598989
rx_queue_6_bytes: 440079884
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 120008
rx_queue_7_bytes: 91041282
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp3s0f1
NIC statistics:
rx_packets: 2417818
tx_packets: 1605247
rx_bytes: 1790337041
tx_bytes: 1268847302
rx_broadcast: 80
tx_broadcast: 11
rx_multicast: 14235
tx_multicast: 463
multicast: 14235
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 511041
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 1790337041
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 187486
tx_queue_0_bytes: 158004014
tx_queue_0_restart: 0
tx_queue_1_packets: 292695
tx_queue_1_bytes: 216338071
tx_queue_1_restart: 0
tx_queue_2_packets: 230556
tx_queue_2_bytes: 187701434
tx_queue_2_restart: 0
tx_queue_3_packets: 175072
tx_queue_3_bytes: 132739732
tx_queue_3_restart: 0
tx_queue_4_packets: 201110
tx_queue_4_bytes: 169995905
tx_queue_4_restart: 0
tx_queue_5_packets: 164302
tx_queue_5_bytes: 122529296
tx_queue_5_restart: 0
tx_queue_6_packets: 176127
tx_queue_6_bytes: 142367535
tx_queue_6_restart: 0
tx_queue_7_packets: 177899
tx_queue_7_bytes: 132745095
tx_queue_7_restart: 0
rx_queue_0_packets: 244981
rx_queue_0_bytes: 175468464
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 226753
rx_queue_1_bytes: 164297931
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 309263
rx_queue_2_bytes: 224912817
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 344819
rx_queue_3_bytes: 256411809
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 413956
rx_queue_4_bytes: 306366538
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 222891
rx_queue_5_bytes: 163512696
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 309235
rx_queue_6_bytes: 234497920
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 345920
rx_queue_7_bytes: 255197594
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp4s0f0
NIC statistics:
rx_packets: 263095022
tx_packets: 165984477
rx_bytes: 358188428243
tx_bytes: 224059860328
rx_broadcast: 0
tx_broadcast: 0
rx_multicast: 14260
tx_multicast: 475
multicast: 14260
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 10618678
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 358188428243
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 24523878
tx_queue_0_bytes: 35555825370
tx_queue_0_restart: 0
tx_queue_1_packets: 22792805
tx_queue_1_bytes: 33160725172
tx_queue_1_restart: 0
tx_queue_2_packets: 32186938
tx_queue_2_bytes: 42938373893
tx_queue_2_restart: 0
tx_queue_3_packets: 27532192
tx_queue_3_bytes: 35219225817
tx_queue_3_restart: 0
tx_queue_4_packets: 12999613
tx_queue_4_bytes: 16197252415
tx_queue_4_restart: 0
tx_queue_5_packets: 20425057
tx_queue_5_bytes: 25328350119
tx_queue_5_restart: 0
tx_queue_6_packets: 13115439
tx_queue_6_bytes: 17344340207
tx_queue_6_restart: 0
tx_queue_7_packets: 12408555
tx_queue_7_bytes: 17651829427
tx_queue_7_restart: 0
rx_queue_0_packets: 28425656
rx_queue_0_bytes: 37636565075
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 8984649
rx_queue_1_bytes: 2071222507
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 36295986
rx_queue_2_bytes: 54200457391
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 23349464
rx_queue_3_bytes: 33468571056
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 5805069
rx_queue_4_bytes: 715544992
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 47433585
rx_queue_5_bytes: 67747439829
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 40708301
rx_queue_6_bytes: 53328721761
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 72092312
rx_queue_7_bytes: 107967525544
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -S enp4s0f1
NIC statistics:
rx_packets: 100608455
tx_packets: 166526987
rx_bytes: 140666343603
tx_bytes: 222986388680
rx_broadcast: 91
tx_broadcast: 8
rx_multicast: 14286
tx_multicast: 469
multicast: 14286
collisions: 0
rx_crc_errors: 0
rx_no_buffer_count: 0
rx_missed_errors: 0
tx_aborted_errors: 0
tx_carrier_errors: 0
tx_window_errors: 0
tx_abort_late_coll: 0
tx_deferred_ok: 0
tx_single_coll_ok: 0
tx_multi_coll_ok: 0
tx_timeout_count: 0
rx_long_length_errors: 0
rx_short_length_errors: 0
rx_align_errors: 0
tx_tcp_seg_good: 9252151
tx_tcp_seg_failed: 0
rx_flow_control_xon: 0
rx_flow_control_xoff: 0
tx_flow_control_xon: 0
tx_flow_control_xoff: 0
rx_long_byte_count: 140666343603
tx_dma_out_of_sync: 0
tx_smbus: 0
rx_smbus: 0
dropped_smbus: 0
os2bmc_rx_by_bmc: 0
os2bmc_tx_by_bmc: 0
os2bmc_tx_by_host: 0
os2bmc_rx_by_host: 0
tx_hwtstamp_timeouts: 0
tx_hwtstamp_skipped: 0
rx_hwtstamp_cleared: 0
rx_errors: 0
tx_errors: 0
tx_dropped: 0
rx_length_errors: 0
rx_over_errors: 0
rx_frame_errors: 0
rx_fifo_errors: 0
tx_fifo_errors: 0
tx_heartbeat_errors: 0
tx_queue_0_packets: 9639934
tx_queue_0_bytes: 13277298005
tx_queue_0_restart: 0
tx_queue_1_packets: 21458007
tx_queue_1_bytes: 28789498862
tx_queue_1_restart: 0
tx_queue_2_packets: 22103147
tx_queue_2_bytes: 28134369759
tx_queue_2_restart: 0
tx_queue_3_packets: 26218962
tx_queue_3_bytes: 32565488967
tx_queue_3_restart: 0
tx_queue_4_packets: 26578778
tx_queue_4_bytes: 39751527579
tx_queue_4_restart: 0
tx_queue_5_packets: 24551167
tx_queue_5_bytes: 36050216125
tx_queue_5_restart: 0
tx_queue_6_packets: 18808095
tx_queue_6_bytes: 27305113278
tx_queue_6_restart: 0
tx_queue_7_packets: 17168897
tx_queue_7_bytes: 16446768013
tx_queue_7_restart: 0
rx_queue_0_packets: 25074819
rx_queue_0_bytes: 37499171327
rx_queue_0_drops: 0
rx_queue_0_csum_err: 0
rx_queue_0_alloc_failed: 0
rx_queue_1_packets: 2573243
rx_queue_1_bytes: 780414215
rx_queue_1_drops: 0
rx_queue_1_csum_err: 0
rx_queue_1_alloc_failed: 0
rx_queue_2_packets: 35512962
rx_queue_2_bytes: 53302135608
rx_queue_2_drops: 0
rx_queue_2_csum_err: 0
rx_queue_2_alloc_failed: 0
rx_queue_3_packets: 3976283
rx_queue_3_bytes: 679395976
rx_queue_3_drops: 0
rx_queue_3_csum_err: 0
rx_queue_3_alloc_failed: 0
rx_queue_4_packets: 24670023
rx_queue_4_bytes: 36883071881
rx_queue_4_drops: 0
rx_queue_4_csum_err: 0
rx_queue_4_alloc_failed: 0
rx_queue_5_packets: 3467062
rx_queue_5_bytes: 4846515985
rx_queue_5_drops: 0
rx_queue_5_csum_err: 0
rx_queue_5_alloc_failed: 0
rx_queue_6_packets: 4683116
rx_queue_6_bytes: 5428970649
rx_queue_6_drops: 0
rx_queue_6_csum_err: 0
rx_queue_6_alloc_failed: 0
rx_queue_7_packets: 650947
rx_queue_7_bytes: 844234142
rx_queue_7_drops: 0
rx_queue_7_csum_err: 0
rx_queue_7_alloc_failed: 0
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp3s0f0
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:03:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp3s0f1
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:03:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp4s0f0
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:04:00.0
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ ethtool -i enp4s0f1
driver: igb
version: 5.4.0-k
firmware-version: 1.0.1
expansion-rom-version:
bus-info: 0000:04:00.1
supports-statistics: yes
supports-test: yes
supports-eeprom-access: yes
supports-register-dump: yes
supports-priv-flags: yes
cephuser@cephnode6:~$
cephuser@cephnode6:~$ dmesg -TL | grep igb
[Thu Jul 18 10:36:09 2019] igb: Intel(R) Gigabit Ethernet Network Driver - version 5.4.0-k
[Thu Jul 18 10:36:09 2019] igb: Copyright (c) 2007-2014 Intel Corporation.
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: added PHC on eth0
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: eth0: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:20
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: eth0: PBA No: Unknown
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: added PHC on eth1
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: eth1: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:21
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: eth1: PBA No: Unknown
[Thu Jul 18 10:36:10 2019] igb 0000:03:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: added PHC on eth2
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: eth2: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:22
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: eth2: PBA No: Unknown
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: added PHC on eth3
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: Intel(R) Gigabit Ethernet Network Connection
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: eth3: (PCIe:2.5Gb/s:Width x4) 00:25:90:eb:1c:23
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: eth3: PBA No: Unknown
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1: Using MSI-X interrupts. 8 rx queue(s), 8 tx queue(s)
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.1 enp4s0f1: renamed from eth3
[Thu Jul 18 10:36:11 2019] igb 0000:03:00.0 enp3s0f0: renamed from eth0
[Thu Jul 18 10:36:11 2019] igb 0000:03:00.1 enp3s0f1: renamed from eth1
[Thu Jul 18 10:36:11 2019] igb 0000:04:00.0 enp4s0f0: renamed from eth2
[Thu Jul 18 10:36:18 2019] igb 0000:04:00.1 enp4s0f1: igb: enp4s0f1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:18 2019] igb 0000:03:00.0 enp3s0f0: igb: enp3s0f0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:19 2019] igb 0000:04:00.0 enp4s0f0: igb: enp4s0f0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
[Thu Jul 18 10:36:19 2019] igb 0000:03:00.1 enp3s0f1: igb: enp3s0f1 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: RX
cephuser@cephnode6:~$
cephuser@cephnode6:~$ uptime
14:39:44 up 4:03, 1 user, load average: 1.84, 1.59, 1.57
cephuser@cephnode6:~$
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
