Attached are my present spec files for both servers (which are also my
clients); both machines use the same files.
To trigger the memory leak, all I need to do is a "dd if=/phys/blah0
of=/dev/null bs=1M", where /phys/blah0 was created with "dd if=/dev/zero
of=/phys/blah0 bs=10M count=1024". It seems fine for a moment, but then
glusterfs starts growing and growing until it can't claim any more RAM
and becomes useless (can't ls the mountpoint, dd doesn't complete)...
Both machines are running Ubuntu Edgy + Fuse 2.6.3.
Thanks,
Brent
On Fri, 9 Mar 2007, Anand Avati wrote:
Brent,
we are unable to get the memory leak in read-ahead (we found one and
fixed, but your mail indicateds there is another). can you please mail
your exact volume spec files and the command line to initiate the I/O
which reproduces the leak?
regards,
avati
On Fri, Mar 09, 2007 at 07:25:03PM -0500, Brent A Nelson wrote:
I just confirmed it on client-side, as well. glusterfs was stable for a
minute, and then it suddenly started increasing in size very rapidly.
Thanks,
Brent
volume disk0
type storage/posix # POSIX FS translator
option directory /share0 # Export this directory
end-volume
volume disk1
type storage/posix # POSIX FS translator
option directory /share1 # Export this directory
end-volume
volume disk2
type storage/posix # POSIX FS translator
option directory /share2 # Export this directory
end-volume
volume disk3
type storage/posix # POSIX FS translator
option directory /share3 # Export this directory
end-volume
volume disk4
type storage/posix # POSIX FS translator
option directory /share4 # Export this directory
end-volume
volume disk5
type storage/posix # POSIX FS translator
option directory /share5 # Export this directory
end-volume
volume disk6
type storage/posix # POSIX FS translator
option directory /share6 # Export this directory
end-volume
volume disk7
type storage/posix # POSIX FS translator
option directory /share7 # Export this directory
end-volume
volume disk8
type storage/posix # POSIX FS translator
option directory /share8 # Export this directory
end-volume
volume disk9
type storage/posix # POSIX FS translator
option directory /share9 # Export this directory
end-volume
volume disk10
type storage/posix # POSIX FS translator
option directory /share10 # Export this directory
end-volume
volume disk11
type storage/posix # POSIX FS translator
option directory /share11 # Export this directory
end-volume
volume disk12
type storage/posix # POSIX FS translator
option directory /share12 # Export this directory
end-volume
volume disk13
type storage/posix # POSIX FS translator
option directory /share13 # Export this directory
end-volume
volume disk14
type storage/posix # POSIX FS translator
option directory /share14 # Export this directory
end-volume
volume disk15
type storage/posix # POSIX FS translator
option directory /share15 # Export this directory
end-volume
volume share0
type performance/io-threads
option thread-count 8
subvolumes disk0
end-volume
volume share1
type performance/io-threads
option thread-count 8
subvolumes disk1
end-volume
volume share2
type performance/io-threads
option thread-count 8
subvolumes disk2
end-volume
volume share3
type performance/io-threads
option thread-count 8
subvolumes disk3
end-volume
volume share4
type performance/io-threads
option thread-count 8
subvolumes disk4
end-volume
volume share5
type performance/io-threads
option thread-count 8
subvolumes disk5
end-volume
volume share6
type performance/io-threads
option thread-count 8
subvolumes disk6
end-volume
volume share7
type performance/io-threads
option thread-count 8
subvolumes disk7
end-volume
volume share8
type performance/io-threads
option thread-count 8
subvolumes disk8
end-volume
volume share9
type performance/io-threads
option thread-count 8
subvolumes disk9
end-volume
volume share10
type performance/io-threads
option thread-count 8
subvolumes disk10
end-volume
volume share11
type performance/io-threads
option thread-count 8
subvolumes disk11
end-volume
volume share12
type performance/io-threads
option thread-count 8
subvolumes disk12
end-volume
volume share13
type performance/io-threads
option thread-count 8
subvolumes disk13
end-volume
volume share14
type performance/io-threads
option thread-count 8
subvolumes disk14
end-volume
volume share15
type performance/io-threads
option thread-count 8
subvolumes disk15
end-volume
volume server
type protocol/server
option transport-type tcp/server # For TCP/IP transport
# option transport-type ib-sdp/server # For Infiniband transport
# option bind-address 192.168.1.10 # Default is to listen on all interfaces
# option listen-port 6996 # Default is 6996
option client-volume-filename /etc/glusterfs/glusterfs-client.vol
subvolumes share0 share1 share2 share3 share4 share5 share6 share7 share8 share9 share10 share11 share12 share13 share14 share15
# NOTE: Access to any volume through protocol/server is denied by
# default. You need to explicitly grant access through # "auth"
# option.
option auth.ip.share0.allow 128.227.64.*,128.227.89.*
option auth.ip.share1.allow 128.227.64.*,128.227.89.*
option auth.ip.share2.allow 128.227.64.*,128.227.89.*
option auth.ip.share3.allow 128.227.64.*,128.227.89.*
option auth.ip.share4.allow 128.227.64.*,128.227.89.*
option auth.ip.share5.allow 128.227.64.*,128.227.89.*
option auth.ip.share6.allow 128.227.64.*,128.227.89.*
option auth.ip.share7.allow 128.227.64.*,128.227.89.*
option auth.ip.share8.allow 128.227.64.*,128.227.89.*
option auth.ip.share9.allow 128.227.64.*,128.227.89.*
option auth.ip.share10.allow 128.227.64.*,128.227.89.*
option auth.ip.share11.allow 128.227.64.*,128.227.89.*
option auth.ip.share12.allow 128.227.64.*,128.227.89.*
option auth.ip.share13.allow 128.227.64.*,128.227.89.*
option auth.ip.share14.allow 128.227.64.*,128.227.89.*
option auth.ip.share15.allow 128.227.64.*,128.227.89.*
end-volume
volume share0-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share0 # name of the remote volume
end-volume
volume share0-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share0 # name of the remote volume
end-volume
volume share1-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share1 # name of the remote volume
end-volume
volume share1-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share1 # name of the remote volume
end-volume
volume share2-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share2 # name of the remote volume
end-volume
volume share2-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share2 # name of the remote volume
end-volume
volume share3-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share3 # name of the remote volume
end-volume
volume share3-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share3 # name of the remote volume
end-volume
volume share4-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share4 # name of the remote volume
end-volume
volume share4-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share4 # name of the remote volume
end-volume
volume share5-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share5 # name of the remote volume
end-volume
volume share5-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share5 # name of the remote volume
end-volume
volume share6-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share6 # name of the remote volume
end-volume
volume share6-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share6 # name of the remote volume
end-volume
volume share7-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share7 # name of the remote volume
end-volume
volume share7-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share7 # name of the remote volume
end-volume
volume share8-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share8 # name of the remote volume
end-volume
volume share8-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share8 # name of the remote volume
end-volume
volume share9-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share9 # name of the remote volume
end-volume
volume share9-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share9 # name of the remote volume
end-volume
volume share10-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share10 # name of the remote volume
end-volume
volume share10-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share10 # name of the remote volume
end-volume
volume share11-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share11 # name of the remote volume
end-volume
volume share11-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share11 # name of the remote volume
end-volume
volume share12-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share12 # name of the remote volume
end-volume
volume share12-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share12 # name of the remote volume
end-volume
volume share13-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share13 # name of the remote volume
end-volume
volume share13-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share13 # name of the remote volume
end-volume
volume share14-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share14 # name of the remote volume
end-volume
volume share14-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share14 # name of the remote volume
end-volume
volume share15-0
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.241 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share15 # name of the remote volume
end-volume
volume share15-1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
# option transport-type ib-sdp/client # for Infiniband transport
option remote-host 128.227.64.242 # IP address of the remote brick
# option remote-port 6996 # default server port is 6996
option remote-subvolume share15 # name of the remote volume
end-volume
volume mirror0
type cluster/afr
subvolumes share0-0 share0-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror1
type cluster/afr
subvolumes share1-1 share1-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror2
type cluster/afr
subvolumes share2-0 share2-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror3
type cluster/afr
subvolumes share3-1 share3-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror4
type cluster/afr
subvolumes share4-0 share4-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror5
type cluster/afr
subvolumes share5-1 share5-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror6
type cluster/afr
subvolumes share6-0 share6-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror7
type cluster/afr
subvolumes share7-1 share7-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror8
type cluster/afr
subvolumes share8-0 share8-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror9
type cluster/afr
subvolumes share9-1 share9-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror10
type cluster/afr
subvolumes share10-0 share10-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror11
type cluster/afr
subvolumes share11-1 share11-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror12
type cluster/afr
subvolumes share12-0 share12-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror13
type cluster/afr
subvolumes share13-1 share13-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror14
type cluster/afr
subvolumes share14-0 share14-1
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirror15
type cluster/afr
subvolumes share15-1 share15-0
option replicate *:2 # Do not leave space before or after "," and ":"
end-volume
volume mirrors
type cluster/unify
subvolumes mirror0 mirror1 mirror2 mirror3 mirror4 mirror5 mirror6 mirror7 mirror8 mirror9 mirror10 mirror11 mirror12 mirror13 mirror14 mirror15
# option scheduler rr
option scheduler alu
option alu.order disk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
option alu.disk-usage.entry-threshold 2GB
option alu.disk-usage.exit-threshold 60MB
option alu.limits.min-free-disk 5GB
option alu.stat-refresh.interval 10sec
end-volume
volume statprefetch
type performance/stat-prefetch
option cache-seconds 2
subvolumes mirrors
end-volume
volume writebehind
type performance/write-behind
option aggregate-size 131072 # in bytes
subvolumes statprefetch
end-volume
volume readahead
type performance/read-ahead
option page-size 65536 ### in bytes
option page-count 16 ### memory cache size is page-count x page-size per file
subvolumes writebehind
end-volume
