That is an EN card, so it won't do pure Infiniband. The VPI variant is the one that does Ethernet or Infiniband. The EN just does the Ethernet. It does however do RoCE and RDMA over RoCE. Make sure you installed the correct OFED from Mellanox. They have EN and VPI variants. They really aren't interchangeable. See: https://docs.mellanox.com/display/CX4LxEN/Troubleshooting The OFED installation will update the firmware. I'm not sure I'd mess with the standalone MFT. Also see: https://docs.mellanox.com/display/CX4LxEN/Linux+Driver+Installation#LinuxDriverInstallation-DownloadingMellanoxOFED -----Original Message----- From: linux-rdma-owner@xxxxxxxxxxxxxxx <linux-rdma-owner@xxxxxxxxxxxxxxx> On Behalf Of Jianchao Wang Sent: Monday, July 1, 2019 7:04 AM To: linux-rdma@xxxxxxxxxxxxxxx Cc: leon@xxxxxxxxxx; dledford@xxxxxxxxxx; jgg@xxxxxxxx Subject: [EXT] RDMA on Mellanox ConnectX-4 Lx cannot work Hi RDMA guys This is my first time to try to setup a rdma environment. Would anyone please give some comment here and many thanks in advance. I have nodes with a Mellanox CX4121A card as following 01:00.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4 Lx] 01:00.1 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4 Lx] It could work well with normal tcp/ip on it. But the rdma doesn't. The mlx5_ib module is loaded, but /sys/class/infiniband is empty [root@node91 eth2]# lsmod | grep mlx5 mlx5_ib 123666 0 ib_core 98787 15 rdma_cm,ib_cm,ib_sa,iw_cm,xprtrdma,mlx5_ib,ib_mad,ib_srp,ib_ucm,ib_iser,ib_srpt,ib_umad,ib_uverbs,ib_ipoib,ib_isert mlx5_core 160250 1 mlx5_ib And here is the ouput of mstconfig. There is _no_ LINK_TYPE_P1, does it mean this is a Ethernet-only card or I just miss some configuration ? Device #1: ---------- Device type: ConnectX4LX Name: N/A Description: N/A Device: 01:00.1 Configurations: Next Boot MEMIC_BAR_SIZE 0 MEMIC_SIZE_LIMIT _256KB(1) ROCE_NEXT_PROTOCOL 254 NON_PREFETCHABLE_PF_BAR False(0) NUM_OF_VFS 8 SRIOV_EN True(1) PF_LOG_BAR_SIZE 5 VF_LOG_BAR_SIZE 0 NUM_PF_MSIX 63 NUM_VF_MSIX 11 INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) SW_RECOVERY_ON_ERRORS False(0) RESET_WITH_HOST_ON_ERRORS False(0) CQE_COMPRESSION BALANCED(0) IP_OVER_VXLAN_EN False(0) LRO_LOG_TIMEOUT0 6 LRO_LOG_TIMEOUT1 7 LRO_LOG_TIMEOUT2 8 LRO_LOG_TIMEOUT3 13 LOG_DCR_HASH_TABLE_SIZE 14 DCR_LIFO_SIZE 16384 ROCE_CC_PRIO_MASK_P1 255 ROCE_CC_ALGORITHM_P1 ECN(0) ROCE_CC_PRIO_MASK_P2 255 ROCE_CC_ALGORITHM_P2 ECN(0) CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) CLAMP_TGT_RATE_P1 False(0) RPG_TIME_RESET_P1 300 RPG_BYTE_RESET_P1 32767 RPG_THRESHOLD_P1 1 RPG_MAX_RATE_P1 0 RPG_AI_RATE_P1 5 RPG_HAI_RATE_P1 50 RPG_GD_P1 11 RPG_MIN_DEC_FAC_P1 50 RPG_MIN_RATE_P1 1 RATE_TO_SET_ON_FIRST_CNP_P1 0 DCE_TCP_G_P1 1019 DCE_TCP_RTT_P1 1 RATE_REDUCE_MONITOR_PERIOD_P1 4 INITIAL_ALPHA_VALUE_P1 1023 MIN_TIME_BETWEEN_CNPS_P1 0 CNP_802P_PRIO_P1 6 CNP_DSCP_P1 48 CLAMP_TGT_RATE_AFTER_TIME_INC_P2 True(1) CLAMP_TGT_RATE_P2 False(0) RPG_TIME_RESET_P2 300 RPG_BYTE_RESET_P2 32767 RPG_THRESHOLD_P2 1 RPG_MAX_RATE_P2 0 RPG_AI_RATE_P2 5 RPG_HAI_RATE_P2 50 RPG_GD_P2 11 RPG_MIN_DEC_FAC_P2 50 RPG_MIN_RATE_P2 1 RATE_TO_SET_ON_FIRST_CNP_P2 0 DCE_TCP_G_P2 1019 DCE_TCP_RTT_P2 1 RATE_REDUCE_MONITOR_PERIOD_P2 4 INITIAL_ALPHA_VALUE_P2 1023 MIN_TIME_BETWEEN_CNPS_P2 0 CNP_802P_PRIO_P2 6 CNP_DSCP_P2 48 LLDP_NB_DCBX_P1 False(0) LLDP_NB_RX_MODE_P1 OFF(0) LLDP_NB_TX_MODE_P1 OFF(0) LLDP_NB_DCBX_P2 False(0) LLDP_NB_RX_MODE_P2 OFF(0) LLDP_NB_TX_MODE_P2 OFF(0) DCBX_IEEE_P1 True(1) DCBX_CEE_P1 True(1) DCBX_WILLING_P1 True(1) DCBX_IEEE_P2 True(1) DCBX_CEE_P2 True(1) DCBX_WILLING_P2 True(1) KEEP_ETH_LINK_UP_P1 True(1) KEEP_IB_LINK_UP_P1 False(0) KEEP_LINK_UP_ON_BOOT_P1 False(0) KEEP_LINK_UP_ON_STANDBY_P1 False(0) KEEP_ETH_LINK_UP_P2 True(1) KEEP_IB_LINK_UP_P2 False(0) KEEP_LINK_UP_ON_BOOT_P2 False(0) KEEP_LINK_UP_ON_STANDBY_P2 False(0) NUM_OF_VL_P1 _4_VLs(3) NUM_OF_TC_P1 _8_TCs(0) NUM_OF_PFC_P1 8 NUM_OF_VL_P2 _4_VLs(3) NUM_OF_TC_P2 _8_TCs(0) NUM_OF_PFC_P2 8 DUP_MAC_ACTION_P1 LAST_CFG(0) SRIOV_IB_ROUTING_MODE_P1 True(1) IB_ROUTING_MODE_P1 LID(1) DUP_MAC_ACTION_P2 LAST_CFG(0) SRIOV_IB_ROUTING_MODE_P2 True(1) IB_ROUTING_MODE_P2 LID(1) PCI_WR_ORDERING per_mkey(0) MULTI_PORT_VHCA_EN False(0) PORT_OWNER True(1) ALLOW_RD_COUNTERS True(1) RENEG_ON_CHANGE True(1) TRACER_ENABLE True(1) IP_VER IPv4(0) UEFI_HII_EN False(0) BOOT_VLAN 1 LEGACY_BOOT_PROTOCOL PXE(1) BOOT_RETRY_CNT NONE(0) BOOT_LACP_DIS True(1) BOOT_VLAN_EN False(0) BOOT_PKEY 0 BOOT_DBG_LOG False(0) UEFI_LOGS DISABLED(0) EXP_ROM_UEFI_ARM_ENABLE False(0) EXP_ROM_UEFI_x86_ENABLE False(0) EXP_ROM_PXE_ENABLE True(1) ADVANCED_PCI_SETTINGS False(0) Linux node91 3.10.0-327.el7.centos.scst72.x86_64 #1 SMP Thu Jan 10 09:33:22 CST 2019 x86_64 x86_64 x86_64 GNU/Linux CONFIDENTIALITY NOTICE: This email and any attachments are for the sole use of the intended recipient and may contain material that is proprietary, confidential, privileged or otherwise legally protected or restricted under applicable government laws. Any review, disclosure, distributing or other use without expressed permission of the sender is strictly prohibited. If you are not the intended recipient, please contact the sender and delete all copies without reading, printing, or saving.
