Can anyone offer any help with infiniband? I can't find much at all on how
to configure server configs with ib-sdp or ib-verbs. I have basic
infiniband working:
root@xxxxxxxxxxxxxxx# ibhosts
Ca : 0x0002c90200001884 ports 2 "vs2 HCA-1"
Ca : 0x0005ad00000505f8 ports 2 "vs0 HCA-1"
Ca : 0x00066a0098003fe2 ports 2 "vs1 HCA-1"
I am not sure if IP over IB is required, but I turned that on and can ping
over infiniband also. I am not quite sure how the addressing works if you
dont use IP over IB.
I have tried the config below that works fine with tcp with ib-sdp and
ip-verbs and no matter what I try I just get connection refused. Do I need
to use the 0x0002c90200001884 addresses instead of the server IP addresses?
I have also tried using the ethernet hosts IPs as well as the infiniband
ib0 IPs.
# Server 0
volume brick-a-ns
type protocol/client
option transport-type ib-sdp/client
option remote-host 192.168.0.10
option transport-timeout 2
option remote-subvolume brick-a-ns
end-volume
volume brick-a
type protocol/client
option transport-type ib-sdp/client
option remote-host 192.168.0.10
option transport-timeout 2
option remote-subvolume brick-a
end-volume
volume mirror-c
type protocol/client
option transport-type ib-sdp/client
option remote-host 192.168.0.10
option transport-timeout 2
option remote-subvolume mirror-c
end-volume
# Server 1
volume brick-b-ns
type storage/posix
option directory /ns
end-volume
volume brick-b
type storage/posix
option directory /raid/brick-b
end-volume
volume mirror-a
type storage/posix
option directory /raid/mirror-a
end-volume
# Server 2
volume brick-c-ns
type protocol/client
option transport-type ib-sdp/client
option remote-host 192.168.0.12
option transport-timeout 2
option remote-subvolume brick-c-ns
end-volume
volume brick-c
type protocol/client
option transport-type ib-sdp/client
option remote-host 192.168.0.12
option transport-timeout 2
option remote-subvolume brick-c
end-volume
volume mirror-b
type protocol/client
option transport-type ib-sdp/client
option remote-host 192.168.0.12
option transport-timeout 2
option remote-subvolume mirror-b
end-volume
# NameSpace AFR
volume block-ns-afr
type cluster/afr
subvolumes brick-a-ns brick-b-ns brick-c-ns
option replicate *:3
option self-heal on
end-volume
# Block AFR
volume block-a-afr
type cluster/afr
subvolumes brick-a mirror-a
option replicate *:2
option self-heal on
end-volume
volume block-b-afr
type cluster/afr
subvolumes brick-b mirror-b
option replicate *:2
option self-heal on
end-volume
volume block-c-afr
type cluster/afr
subvolumes brick-c mirror-c
option replicate *:2
option self-heal on
end-volume
# Unify Blocks into Share
volume share-unify
type cluster/unify
subvolumes block-a-afr block-b-afr block-c-afr
option namespace block-ns-afr
option scheduler rr
end-volume
# Final Share with performance
volume share
type performance/io-threads
option thread-count 12
option cache-size 32MB
subvolumes share-unify
end-volume
# Server settings
volume server
type protocol/server
option transport-type ib-sdp/server
subvolumes share
option auth.ip.brick-b-ns.allow 192.168.0.*,127.0.0.1
option auth.ip.brick-b.allow 192.168.0.*,127.0.0.1
option auth.ip.mirror-a.allow 192.168.0.*,127.0.0.1
option auth.ip.share.allow *
end-volume
