2.5 patch 261 crashing with large rsync

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



I have 3 large rsyncs that run in parallel that check 5tb of files. It crashes towards the end. During this glusterfs is taking 600megz of memory.

p.s. if you guys want 3 remote machines to work on let me know.

gdb bt:
#0 0xb76490cd in unify_sh_readdir_cbk (frame=0xd5e2de8, cookie=0x8057e08, this=0x80580b8, op_ret=-1, op_errno=107, entry=0x0, count=0) at unify-self-heal.c:218 #1 0xb7652ec4 in client_readdir_cbk (frame=0xd50d0f0, args=0xe16e860) at client-protocol.c:2816 #2 0xb7655e65 in client_protocol_cleanup (trans=0x8090200) at client-protocol.c:4089 #3 0xb7656bd7 in notify (this=0x8057e08, event=3, data=0x8090200) at client-protocol.c:4515 #4 0xb7fca4a7 in transport_notify (this=0x8090200, event=17) at transport.c:152
#5  0xb7fcab8c in epoll_notify (eevent=17, data=0x8090200) at epoll.c:54
#6  0xb7fcae74 in sys_epoll_iteration (ctx=0xbffedd0c) at epoll.c:146
#7  0xb7fca69a in poll_iteration (ctx=0xbffedd0c) at transport.c:260
#8  0x0804a6de in main (argc=3, argv=0xbffeddd4) at glusterfs.c:341

=========================
glusterfs.log
2007-07-02 19:01:29 C [client-protocol.c:215:call_bail] server1: bailing transport 2007-07-02 19:01:29 E [client-protocol.c:328:client_protocol_xfer] server1: transport_submit failed 2007-07-02 19:01:29 C [client-protocol.c:215:call_bail] server1vol1: bailing transport 2007-07-02 19:01:29 C [tcp.c:81:tcp_disconnect] server1: connection disconnected 2007-07-02 19:01:29 E [client-protocol.c:328:client_protocol_xfer] server1vol1: transport_submit failed 2007-07-02 19:01:29 C [common-utils.c:205:gf_print_trace] debug-backtrace: Got signal (11), printing backtrace 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/libglusterfs.so.0(gf_print_trace+0x2b) [0xb7fc8a8a] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: [0xffffe420] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/glusterfs/1.3.0-pre5/xlator/protocol/client.so [0xb7652ec4] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/glusterfs/1.3.0-pre5/xlator/protocol/client.so [0xb7655e65] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/glusterfs/1.3.0-pre5/xlator/protocol/client.so(notify+0x131) [0xb7656bd7] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/libglusterfs.so.0(transport_notify+0x6a) [0xb7fca4a7] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/libglusterfs.so.0 [0xb7fcab8c] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/libglusterfs.so.0(sys_epoll_iteration+0x14b) [0xb7fcae74] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/libglusterfs.so.0(poll_iteration+0x1d) [0xb7fca69a] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: [glusterfs] [0x804a6de] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: /lib/tls/i686/cmov/libc.so.6(__libc_start_main+0xd2) [0xb7e78ea2] 2007-07-02 19:01:29 C [common-utils.c:207:gf_print_trace] debug-backtrace: [glusterfs] [0x8049cdd]

=========================
server.vol:
volume volume1
 type storage/posix
 option directory /volume1
end-volume

volume clusterfs1
  type performance/io-threads
  option thread-count 10
  subvolumes volume1
end-volume

#######

volume volumenamespace
 type storage/posix
 option directory /volume.namespace
end-volume

###

volume clusterfs
 type protocol/server
 option transport-type tcp/server
 subvolumes clusterfs1 volumenamespace
 option auth.ip.clusterfs1.allow *
 option auth.ip.volumenamespace.allow *
end-volume

=========================
client.vol
volume server1
        type protocol/client
        option transport-type tcp/client     # for TCP/IP transport
        option remote-host 127.0.0.1     # IP address of the remote brick
        option remote-subvolume volumenamespace
end-volume

volume server1vol1
        type protocol/client
        option transport-type tcp/client     # for TCP/IP transport
        option remote-host 127.0.0.1     # IP address of the remote brick
        option remote-subvolume clusterfs1
end-volume

#volume server1vol1
  #type performance/io-threads
  #option thread-count 10
  #subvolumes server1vol1spec
#end-volume

###################

volume bricks
 type cluster/unify
 option namespace server1
 option readdir-force-success on  # ignore failed mounts
 subvolumes server1vol1

 option scheduler rr
 option rr.limits.min-free-disk 5 #%
end-volume

volume writebehind   #writebehind improves write performance a lot
 type performance/write-behind
 option aggregate-size 131072 # in bytes
 subvolumes bricks
end-volume


volume readahead
 type performance/read-ahead
 option page-size 65536     # unit in bytes
 option page-count 16       # cache per file  = (page-count x page-size)
 subvolumes writebehind
end-volume





[Index of Archives]     [Gluster Users]     [Ceph Users]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Security]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [eCos]     [Asterisk Internet PBX]     [Linux API]

  Powered by Linux