Hi everyone,
Here is a template for a HA storage configuration with GlusterFS.
Configuration is :
2 nodes on a network A (or2 and or3)
1 node on a network B (op1)
We want to provide HA services for both networks A & B
In this example, we also provide a "backup" node for network A
All clients from networks A & B should see the same volume at every time
################ Server 1 ################
volume tbs-clust-op1-local-data
type storage/posix
option directory /export/GLUSTER/data
end-volume
volume tbs-clust-op1-local-data-locks
type features/posix-locks
subvolumes tbs-clust-op1-local-data
end-volume
volume tbs-clust-op1-local-ns
type storage/posix
option directory /export/GLUSTER/namespace
end-volume
volume tbs-clust-or2-remote-data
type protocol/client
option transport-type tcp/client
option remote-host 192.168.28.6
option remote-subvolume tbs-clust-or2-local-data-locks
end-volume
volume tbs-clust-or3-remote-data
type protocol/client
option transport-type tcp/client
option remote-host 192.168.28.7
option remote-subvolume tbs-clust-or3-local-data-locks
end-volume
### Performance xlators
volume tbs-clust-or2-iot
type performance/io-threads
option thread-count 2
subvolumes tbs-clust-or2-remote-data
end-volume
volume tbs-clust-or2-wb
type performance/write-behind
subvolumes tbs-clust-or2-iot
end-volume
volume tbs-clust-or3-iot
type performance/io-threads
option thread-count 2
subvolumes tbs-clust-or3-remote-data
end-volume
volume tbs-clust-or3-wb
type performance/write-behind
subvolumes tbs-clust-or3-iot
end-volume
### Replication on server-side
volume tbs-clust-data-afr
type cluster/afr
subvolumes tbs-clust-op1-local-data-locks tbs-clust-or3-wb
tbs-clust-or2-wb
option replicate *:3
option self-heal on
end-volume
### Unification on server-side
volume tbs-clust-unify
type cluster/unify
subvolumes tbs-clust-data-afr
option namespace tbs-clust-op1-local-ns
option scheduler rr
end-volume
### Global Performance xlators
volume tbs-clust-server
type performance/io-threads
option thread-count 8
option cache-size 32MB
subvolumes tbs-clust-unify
end-volume
### Volumes exportation
volume server
type protocol/server
option transport-type tcp/server
subvolumes tbs-clust-server
option auth.ip.tbs-clust-op1-local-data-locks.allow
127.0.0.1,10.24.133.178,192.168.28.6,192.168.28.7,192.168.28.5
option auth.ip.tbs-clust-op1-local-ns.allow 127.0.0.1,10.24.133.178
option auth.ip.tbs-clust-server.allow
127.0.0.1,10.24.133.178,192.168.28.6,192.168.28.7,192.168.28.5
end-volume
################ Server 2 ################
volume tbs-clust-or2-local-data
type storage/posix
option directory /export/GLUSTER/data
end-volume
volume tbs-clust-or2-local-data-locks
type features/posix-locks
subvolumes tbs-clust-or2-local-data
end-volume
volume tbs-clust-or2-local-ns
type storage/posix
option directory /export/GLUSTER/namespace
end-volume
volume tbs-clust-or3-remote-data
type protocol/client
option transport-type tcp/client
option remote-host 192.168.28.7
option remote-subvolume tbs-clust-or3-local-data-locks
end-volume
volume tbs-clust-op1-remote-data
type protocol/client
option transport-type tcp/client
option remote-host 10.24.133.178
option remote-subvolume tbs-clust-op1-local-data-locks
end-volume
### Performance xlators
volume tbs-clust-op1-iot
type performance/io-threads
option thread-count 4
subvolumes tbs-clust-op1-remote-data
end-volume
volume tbs-clust-op1-wb
type performance/write-behind
subvolumes tbs-clust-op1-iot
end-volume
### Replication on server-side
volume tbs-clust-data-afr
type cluster/afr
subvolumes tbs-clust-or2-local-data-locks
tbs-clust-or3-remote-data tbs-clust-op1-wb
option replicate *:3
option self-heal on
end-volume
### Unification on server-side
volume tbs-clust-unify
type cluster/unify
subvolumes tbs-clust-data-afr
option namespace tbs-clust-or2-local-ns
option scheduler rr
end-volume
### Global Performance xlators
volume tbs-clust-server
type performance/io-threads
option thread-count 8
option cache-size 32MB
subvolumes tbs-clust-unify
end-volume
### Volumes exportation
volume server
type protocol/server
option transport-type tcp/server
subvolumes tbs-clust-server
option auth.ip.tbs-clust-or2-local-data-locks.allow
127.0.0.1,192.168.28.6,192.168.28.7,10.24.133.178,192.168.28.5
option auth.ip.tbs-clust-or2-local-ns.allow 127.0.0.1,192.168.28.6
option auth.ip.tbs-clust-server.allow
127.0.0.1,192.168.28.6,192.168.28.7,10.24.133.178,192.168.28.5
end-volume
################ Server 3 ################
volume tbs-clust-or3-local-data
type storage/posix
option directory /export/GLUSTER/data
end-volume
volume tbs-clust-or3-local-data-locks
type features/posix-locks
subvolumes tbs-clust-or3-local-data
end-volume
volume tbs-clust-or3-local-ns
type storage/posix
option directory /export/GLUSTER/namespace
end-volume
volume tbs-clust-or2-remote-data
type protocol/client
option transport-type tcp/client
option remote-host 192.168.28.6
option remote-subvolume tbs-clust-or2-local-data-locks
end-volume
volume tbs-clust-op1-remote-data
type protocol/client
option transport-type tcp/client
option remote-host 10.24.133.178
option remote-subvolume tbs-clust-op1-local-data-locks
end-volume
### Performance xlators
volume tbs-clust-op1-iot
type performance/io-threads
option thread-count 4
subvolumes tbs-clust-op1-remote-data
end-volume
volume tbs-clust-op1-wb
type performance/write-behind
subvolumes tbs-clust-op1-iot
end-volume
### Replication on server-side
volume tbs-clust-data-afr
type cluster/afr
subvolumes tbs-clust-or3-local-data-locks
tbs-clust-or2-remote-data tbs-clust-op1-wb
option replicate *:3
option self-heal on
end-volume
### Unification on server-side
volume tbs-clust-unify
type cluster/unify
subvolumes tbs-clust-data-afr
option namespace tbs-clust-or3-local-ns
option scheduler rr
end-volume
### Global Performance xlators
volume tbs-clust-server
type performance/io-threads
option thread-count 8
option cache-size 32MB
subvolumes tbs-clust-unify
end-volume
### Volumes exportation
volume server
type protocol/server
option transport-type tcp/server
subvolumes tbs-clust-server
option auth.ip.tbs-clust-or3-local-data-locks.allow
127.0.0.1,192.168.28.7,192.168.28.6,10.24.133.178,192.168.28.5
option auth.ip.tbs-clust-or3-local-ns.allow 127.0.0.1,192.168.28.7
option auth.ip.tbs-clust-server.allow
127.0.0.1,192.168.28.7,192.168.28.6,10.24.133.178,192.168.28.5
end-volume
################ client (1, 2, & 3 and even more !) ################
volume tbs-clust
type protocol/client
option transport-type tcp/client # for TCP/IP transport
option remote-host 192.168.28.6 # IP address of the remote
brick
# option remote-host glust-or.pitux.org # rr DNS resolves
192.168.28.6 &
# 192.168.28.7 &
10.24.133.178 if
# there is a weight-balancing
option remote-subvolume tbs-clust-server # name of the remote volume
option transport-timeout 10
end-volume
### Performance xlators
volume tbs-clust-iothreads
type performance/io-threads
option thread-count 8
option cache-size 32MB
subvolumes tbs-clust
end-volume
volume tbs-clust-write-behind
type performance/write-behind
option aggregate-size 131072 # unit in bytes
subvolumes tbs-clust-iothreads
Enjoy !
Sebastien.
