I have 3 large rsyncs that run in parallel that check 5tb of files. It
crashes towards the end. During this glusterfs is taking 600megz of memory.
p.s. if you guys want 3 remote machines to work on let me know.
gdb bt:
#0 0xb76490cd in unify_sh_readdir_cbk (frame=0xd5e2de8,
cookie=0x8057e08, this=0x80580b8, op_ret=-1, op_errno=107, entry=0x0,
count=0) at unify-self-heal.c:218
#1 0xb7652ec4 in client_readdir_cbk (frame=0xd50d0f0, args=0xe16e860)
at client-protocol.c:2816
#2 0xb7655e65 in client_protocol_cleanup (trans=0x8090200) at
client-protocol.c:4089
#3 0xb7656bd7 in notify (this=0x8057e08, event=3, data=0x8090200) at
client-protocol.c:4515
#4 0xb7fca4a7 in transport_notify (this=0x8090200, event=17) at
transport.c:152
#5 0xb7fcab8c in epoll_notify (eevent=17, data=0x8090200) at epoll.c:54
#6 0xb7fcae74 in sys_epoll_iteration (ctx=0xbffedd0c) at epoll.c:146
#7 0xb7fca69a in poll_iteration (ctx=0xbffedd0c) at transport.c:260
#8 0x0804a6de in main (argc=3, argv=0xbffeddd4) at glusterfs.c:341
=========================
glusterfs.log
2007-07-02 19:01:29 C [client-protocol.c:215:call_bail] server1: bailing
transport
2007-07-02 19:01:29 E [client-protocol.c:328:client_protocol_xfer]
server1: transport_submit failed
2007-07-02 19:01:29 C [client-protocol.c:215:call_bail] server1vol1:
bailing transport
2007-07-02 19:01:29 C [tcp.c:81:tcp_disconnect] server1: connection
disconnected
2007-07-02 19:01:29 E [client-protocol.c:328:client_protocol_xfer]
server1vol1: transport_submit failed
2007-07-02 19:01:29 C [common-utils.c:205:gf_print_trace]
debug-backtrace: Got signal (11), printing backtrace
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/libglusterfs.so.0(gf_print_trace+0x2b) [0xb7fc8a8a]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: [0xffffe420]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/glusterfs/1.3.0-pre5/xlator/protocol/client.so
[0xb7652ec4]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/glusterfs/1.3.0-pre5/xlator/protocol/client.so
[0xb7655e65]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace:
/lib/glusterfs/1.3.0-pre5/xlator/protocol/client.so(notify+0x131)
[0xb7656bd7]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/libglusterfs.so.0(transport_notify+0x6a) [0xb7fca4a7]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/libglusterfs.so.0 [0xb7fcab8c]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/libglusterfs.so.0(sys_epoll_iteration+0x14b)
[0xb7fcae74]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/libglusterfs.so.0(poll_iteration+0x1d) [0xb7fca69a]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: [glusterfs] [0x804a6de]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: /lib/tls/i686/cmov/libc.so.6(__libc_start_main+0xd2)
[0xb7e78ea2]
2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace]
debug-backtrace: [glusterfs] [0x8049cdd]
=========================
server.vol:
volume volume1
type storage/posix
option directory /volume1
end-volume
volume clusterfs1
type performance/io-threads
option thread-count 10
subvolumes volume1
end-volume
#######
volume volumenamespace
type storage/posix
option directory /volume.namespace
end-volume
###
volume clusterfs
type protocol/server
option transport-type tcp/server
subvolumes clusterfs1 volumenamespace
option auth.ip.clusterfs1.allow *
option auth.ip.volumenamespace.allow *
end-volume
=========================
client.vol
volume server1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
option remote-host 127.0.0.1 # IP address of the remote brick
option remote-subvolume volumenamespace
end-volume
volume server1vol1
type protocol/client
option transport-type tcp/client # for TCP/IP transport
option remote-host 127.0.0.1 # IP address of the remote brick
option remote-subvolume clusterfs1
end-volume
#volume server1vol1
#type performance/io-threads
#option thread-count 10
#subvolumes server1vol1spec
#end-volume
###################
volume bricks
type cluster/unify
option namespace server1
option readdir-force-success on # ignore failed mounts
subvolumes server1vol1
option scheduler rr
option rr.limits.min-free-disk 5 #%
end-volume
volume writebehind #writebehind improves write performance a lot
type performance/write-behind
option aggregate-size 131072 # in bytes
subvolumes bricks
end-volume
volume readahead
type performance/read-ahead
option page-size 65536 # unit in bytes
option page-count 16 # cache per file = (page-count x page-size)
subvolumes writebehind
end-volume
