On 1/21/21 11:59 AM, Bart Van Assche wrote:
The approach of verifying whether or not an RDMA interface is associated
with the rdma_rxe interface by looking up its parent device is deprecated
and will be removed soon. Hence this patch that uses the rdma link command
instead.
Cc: Jason Gunthorpe <jgg@xxxxxxxxxx>
Cc: Yi Zhang <yi.zhang@xxxxxxxxxx>
Signed-off-by: Bart Van Assche <bvanassche@xxxxxxx>
---
common/multipath-over-rdma | 107 ++++++++++---------------------------
tests/srp/rc | 9 +---
2 files changed, 28 insertions(+), 88 deletions(-)
diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma
index 9d9d2b27af83..b85e31e7bce1 100644
--- a/common/multipath-over-rdma
+++ b/common/multipath-over-rdma
@@ -76,41 +76,9 @@ is_number() {
[ "$1" -eq "0$1" ] 2>/dev/null
}
-# Check whether a device is an RDMA device. An example argument:
-# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0
-is_rdma_device() {
- local d i inode1 inode2
-
- inode1=$(stat -c %i "$1")
- # echo "inode1 = $inode1"
- for i in /sys/class/infiniband/*; do
- d=/sys/class/infiniband/"$(readlink "$i")"
- d=$(dirname "$(dirname "$d")")
- inode2=$(stat -c %i "$d")
- # echo "inode2 = $inode2"
- if [ "$inode1" = "$inode2" ]; then
- return
- fi
- done
- false
-}
-
# Lists RDMA capable network interface names, e.g. ib0 ib1.
rdma_network_interfaces() {
- (
- cd /sys/class/net &&
- for i in *; do
- [ -e "$i" ] || continue
- # Skip IPoIB (ARPHRD_INFINIBAND) network
- # interfaces.
- [ "$(<"$i"/type)" = 32 ] && continue
- [ -L "$i/device" ] || continue
- d=$(readlink "$i/device" 2>/dev/null)
- if [ -n "$d" ] && is_rdma_device "$i/$d"; then
- echo "$i"
- fi
- done
- )
+ rdma link show | sed -n 's,^link[[:blank:]]*\([^/]*\)/.*,\1,p' | sort -u
We should list the network interface here(like eno1), rxe/siw interfaces
will not work for function get_ipv4_addr
# rdma link
link eno1_rxe/1 state ACTIVE physical_state LINK_UP netdev eno1
link eno1_siw/1 state ACTIVE physical_state LINK_UP netdev eno1
# rdma link show | sed -n 's,^link[[:blank:]]*\([^/]*\)/.*,\1,p' | sort -u
eno1_rxe
eno1_siw
for i in $(rdma_network_interfaces)
do
ipv4_addr=$(get_ipv4_addr "$i")
if [ -n "${ipv4_addr}" ]; then
def_traddr=${ipv4_addr}
fi
done
}
# Check whether any stacked block device holds block device $1. If so, echo
@@ -411,47 +379,36 @@ all_primary_gids() {
done
}
-# Check whether or not an rdma_rxe instance has been associated with network
-# interface $1.
-has_rdma_rxe() {
- local f
-
- for f in /sys/class/infiniband/*/parent; do
- if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then
- return 0
- fi
- done
-
- return 1
+# Check whether or not an rdma_rxe or siw instance has been associated with
+# network interface $1.
+has_soft_rdma() {
+ rdma link | grep -q " netdev $1[[:blank:]]*\$"
}
# Load the rdma_rxe or siw kernel module and associate it with all network
# interfaces.
start_soft_rdma() {
+ local type
+
{
if [ -n "$use_siw" ]; then
modprobe siw || return $?
- (
- cd /sys/class/net &&
- for i in *; do
- [ -e "$i" ] || continue
- [ -e "/sys/class/infiniband/${i}_siw" ] && continue
- rdma link add "${i}_siw" type siw netdev "$i" ||
- echo "Failed to bind the siw driver to $i"
- done
- )
+ type=siw
else
modprobe rdma_rxe || return $?
- (
- cd /sys/class/net &&
- for i in *; do
- if [ -e "$i" ] && ! has_rdma_rxe "$i"; then
- echo "$i" > /sys/module/rdma_rxe/parameters/add ||
- echo "Failed to bind the rdma_rxe driver to $i"
- fi
- done
- )
+ type=rxe
fi
+ (
+ cd /sys/class/net &&
+ for i in *; do
+ [ -e "$i" ] || continue
+ [ "$i" = "lo" ] && continue
+ [ "$(<"$i/addr_len")" = 6 ] || continue
+ has_soft_rdma "$i" && continue
+ rdma link add "${i}_$type" type $type netdev "$i" ||
+ echo "Failed to bind the $type driver to $i"
+ done
+ )
} >>"$FULL"
}
@@ -459,27 +416,17 @@ start_soft_rdma() {
# unload the rdma_rxe kernel module.
stop_soft_rdma() {
{
- (
- cd /sys/class/net &&
- for i in *; do
- if [ -e "$i" ] && has_rdma_rxe "$i"; then
- { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \
- 2>/dev/null
- fi
- done
- )
+ rdma link |
+ sed -n 's,^link[[:blank:]]*\([^/]*\)/.* netdev .*,\1,p' |
+ while read -r i; do
+ echo "$i ..."
+ rdma link del "${i}" ||
+ echo "Failed to unbind siw/rxe from ${i}"
+ done
if ! unload_module rdma_rxe 10; then
echo "Unloading rdma_rxe failed"
return 1
fi
- (
- cd /sys/class/net &&
- for i in *_siw; do
- [ -e "$i" ] || continue
- rdma link del "${i}" ||
- echo "Failed to unbind the siw driver from ${i%_siw}"
- done
- )
if ! unload_module siw 10; then
echo "Unloading siw failed"
return 1
diff --git a/tests/srp/rc b/tests/srp/rc
index 1f665a28db66..b8ac9e27a2fd 100755
--- a/tests/srp/rc
+++ b/tests/srp/rc
@@ -142,14 +142,7 @@ do_ib_cm_login() {
}
rdma_dev_to_net_dev() {
- local b d rdma_dev=$1
-
- b=/sys/class/infiniband/$rdma_dev/parent
- if [ -e "$b" ]; then
- echo "$(<"$b")"
- else
- echo "${rdma_dev%_siw}"
- fi
+ rdma link show "$1/1" | sed 's/.* netdev //;s/[[:blank:]]*$//'
}
# Tell the SRP initiator to log in to an SRP target using the RDMA/CM.
