Hi everyone
I have installed 3.3.1-1 from the Debian repository you provide.
I am using a simple 2 node cluster and running in replication mode. The
connection between the nodes is limited to 100MB/sec (that's bits not
bytes!). Usage will be mainly for read access and since there is always
a local copy available [ exactly 2 replicas on exactly 2 machines ] I
expect very fast read performance. Writes are low volume and very
infrequent - performance is not an issue.
Almost everything works as I would expect.
Write speed is limited to 10Mb (bytes) per second which is what I would
expect and is adequate for the application.
But read speed is either super fast or 10Mb/sec. i.e. read operations
take place on the local copy or the remote seemingly at random.
This not the 'small files problem'. I am aware that Gluster must use
network access for stat() etc. This is all about where the data comes
from on a read(). If I do an m5dum on a 200Mb file it takes either half
a second or 18 seconds.
There is an option read-subvolume.
I have tried to understand how this works from the documentation
available and from the few examples on the web.
I have added the option using:
gluster volume set X read-subvolume Y
It has no effect even after stopping and starting the volume,
remounting, restarting gluster servers etc.
What's more I fail to see how this option could ever work at all. The
configuration changes caused by the above command are rolled out to both
nodes - but what is right for one node is exactly the wrong
configuration for the other node.
Configs attached are in /var/lib/glusterd/vols/shared except
glusterd.vol which is in /etc/glusterfs.
Here is the output of the mount command filtered to just the glusterfs
mount:
10.255.255.1:/shared on /gluster/rw/shared type fuse.glusterfs
(rw,relatime,user_id=0,group_id=0,default_permissions,allow_other,max_read=131072)
10.255.255.1 is local to this host.
I would be very thankful if someone can enlighten me. I am obviously
configuring this wrong. I may have missed something important.
Best regards to all
Allan
-------------- next part --------------
volume shared-client-0
type protocol/client
option remote-host 10.255.255.1
option remote-subvolume /gluster/vol/shared1/brick
option transport-type tcp
option username 1bd771cd-3506-48fc-95de-2fd7605cc012
option password cc801d8d-305b-45c4-9820-bb396ce65d9b
end-volume
volume shared-client-1
type protocol/client
option remote-host 10.255.255.2
option remote-subvolume /gluster/vol/shared2/brick
option transport-type tcp
option username 1bd771cd-3506-48fc-95de-2fd7605cc012
option password cc801d8d-305b-45c4-9820-bb396ce65d9b
end-volume
volume shared-replicate-0
type cluster/replicate
option read-subvolume shared-client-1
subvolumes shared-client-0 shared-client-1
end-volume
volume shared-write-behind
type performance/write-behind
subvolumes shared-replicate-0
end-volume
volume shared-read-ahead
type performance/read-ahead
subvolumes shared-write-behind
end-volume
volume shared-io-cache
type performance/io-cache
subvolumes shared-read-ahead
end-volume
volume shared-quick-read
type performance/quick-read
subvolumes shared-io-cache
end-volume
volume shared-md-cache
type performance/md-cache
subvolumes shared-quick-read
end-volume
volume shared
type debug/io-stats
option latency-measurement off
option count-fop-hits off
subvolumes shared-md-cache
end-volume
-------------- next part --------------
volume shared-posix
type storage/posix
option directory /gluster/vol/shared2/brick
option volume-id df592c0a-8190-42d8-b0d5-e50458a5368e
end-volume
volume shared-access-control
type features/access-control
subvolumes shared-posix
end-volume
volume shared-locks
type features/locks
subvolumes shared-access-control
end-volume
volume shared-io-threads
type performance/io-threads
subvolumes shared-locks
end-volume
volume shared-index
type features/index
option index-base /gluster/vol/shared2/brick/.glusterfs/indices
subvolumes shared-io-threads
end-volume
volume shared-marker
type features/marker
option volume-uuid df592c0a-8190-42d8-b0d5-e50458a5368e
option timestamp-file /var/lib/glusterd/vols/shared/marker.tstamp
option xtime off
option quota off
subvolumes shared-index
end-volume
volume /gluster/vol/shared2/brick
type debug/io-stats
option latency-measurement off
option count-fop-hits off
subvolumes shared-marker
end-volume
volume shared-server
type protocol/server
option transport-type tcp
option auth.login./gluster/vol/shared2/brick.allow 1bd771cd-3506-48fc-95de-2fd7605cc012
option auth.login.1bd771cd-3506-48fc-95de-2fd7605cc012.password cc801d8d-305b-45c4-9820-bb396ce65d9b
option auth.addr./gluster/vol/shared2/brick.allow *
subvolumes /gluster/vol/shared2/brick
end-volume
-------------- next part --------------
volume shared-posix
type storage/posix
option directory /gluster/vol/shared1/brick
option volume-id df592c0a-8190-42d8-b0d5-e50458a5368e
end-volume
volume shared-access-control
type features/access-control
subvolumes shared-posix
end-volume
volume shared-locks
type features/locks
subvolumes shared-access-control
end-volume
volume shared-io-threads
type performance/io-threads
subvolumes shared-locks
end-volume
volume shared-index
type features/index
option index-base /gluster/vol/shared1/brick/.glusterfs/indices
subvolumes shared-io-threads
end-volume
volume shared-marker
type features/marker
option volume-uuid df592c0a-8190-42d8-b0d5-e50458a5368e
option timestamp-file /var/lib/glusterd/vols/shared/marker.tstamp
option xtime off
option quota off
subvolumes shared-index
end-volume
volume /gluster/vol/shared1/brick
type debug/io-stats
option latency-measurement off
option count-fop-hits off
subvolumes shared-marker
end-volume
volume shared-server
type protocol/server
option transport-type tcp
option auth.login./gluster/vol/shared1/brick.allow 1bd771cd-3506-48fc-95de-2fd7605cc012
option auth.login.1bd771cd-3506-48fc-95de-2fd7605cc012.password cc801d8d-305b-45c4-9820-bb396ce65d9b
option auth.addr./gluster/vol/shared1/brick.allow *
subvolumes /gluster/vol/shared1/brick
end-volume
-------------- next part --------------
volume shared-client-0
type protocol/client
option remote-host 10.255.255.1
option remote-subvolume /gluster/vol/shared1/brick
option transport-type tcp
end-volume
volume shared-client-1
type protocol/client
option remote-host 10.255.255.2
option remote-subvolume /gluster/vol/shared2/brick
option transport-type tcp
end-volume
volume shared-replicate-0
type cluster/replicate
option read-subvolume shared-client-1
subvolumes shared-client-0 shared-client-1
end-volume
volume shared-write-behind
type performance/write-behind
subvolumes shared-replicate-0
end-volume
volume shared-read-ahead
type performance/read-ahead
subvolumes shared-write-behind
end-volume
volume shared-io-cache
type performance/io-cache
subvolumes shared-read-ahead
end-volume
volume shared-quick-read
type performance/quick-read
subvolumes shared-io-cache
end-volume
volume shared-md-cache
type performance/md-cache
subvolumes shared-quick-read
end-volume
volume shared
type debug/io-stats
option latency-measurement off
option count-fop-hits off
subvolumes shared-md-cache
end-volume
-------------- next part --------------
volume management
type mgmt/glusterd
option working-directory /var/lib/glusterd
option transport-type socket,rdma
option transport.socket.keepalive-time 10
option transport.socket.keepalive-interval 2
option transport.socket.read-fail-log off
end-volume
