everything was working fine, and then i lost access to my file system.
I looked at logs and it appears I lost connectivity to my server bricks,
but i could ping them and telnet to port 6996
They are all on the same subnet, and no network problems and no sort of
glusterfs recovery occurred.
Script started on Sun 01 Jul 2007 02:54:02 AM PDT
/tmp/1 is my glusterfs client mountpoint
[root@dhcp217 1]# ls
[00mls: cannot open directory .: Transport endpoint is not connected
[m]0;root@dhcp217:/tmp/1 [root@dhcp217 1]# cd
/usr/local/var/log/glusterfs/
2007-06-30 08:54:55 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-06-30 08:54:56 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-06-30 08:54:56 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-06-30 08:55:01 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-06-30 09:20:07 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-06-30 09:20:08 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-06-30 09:20:08 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-06-30 09:20:15 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-06-30 09:20:53 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-06-30 09:24:28 E [protocol.c:262:gf_block_unserialize_transport]
libglusterfs/protocol: full_read of header failed: peer (0.0.0.0)
2007-06-30 09:24:28 C [tcp.c:81:tcp_disconnect] glfsd115-250GB-truecrypt:
connection disconnected
2007-06-30 09:24:28 E [client-protocol.c:327:client_protocol_xfer]
glfsd115-250GB-truecrypt: transport_submit failed
2007-06-30 09:24:31 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:24:34 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:24:39 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:24:47 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:25:00 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:25:21 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:25:55 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:26:50 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:28:19 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:30:43 E [tcp-client.c:171:tcp_connect]
glfsd115-250GB-truecrypt: non-blocking connect() returned: 111 (Connection
refused)
2007-06-30 09:32:10 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-06-30 09:32:31 C [common-utils.c:205:gf_print_trace] debug-backtrace:
Got signal (11), printing backtrace
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(gf_print_trace+0x2d) [0x84d94d]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[0xc33420]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(transport_notify+0x37) [0x84ee47]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(sys_epoll_iteration+0xd9) [0x84f969]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(poll_iteration+0x1d) [0x84ef1d]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8049e70]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/lib/libc.so.6(__libc_start_main+0xe0) [0x125f70]
2007-06-30 09:32:31 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8049a41]
2007-06-30 09:35:53 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-06-30 09:49:02 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-06-30 10:15:14 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-07-01 01:54:43 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:43 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:44 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:44 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:45 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:45 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:46 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:46 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in progress - trace: Network is
unreachable
2007-07-01 01:54:50 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENT_POLLERR
2007-07-01 02:52:06 C [common-utils.c:205:gf_print_trace] debug-backtrace:
Got signal (11), printing backtrace
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(gf_print_trace+0x2d) [0xada94d]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[0xf53420]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/glusterfs/1.3.0-pre5/xlator/cluster/afr.so [0xf91225]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(default_lookup+0x85) [0xada795]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(default_lookup+0x85) [0xada795]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x804d3d6]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libfuse.so.2 [0x8ded61]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libfuse.so.2 [0x8df7a3]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libfuse.so.2(fuse_session_process+0x26) [0x8e0fa6]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x804a740]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(transport_notify+0x37) [0xadbe47]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(sys_epoll_iteration+0xd9) [0xadc969]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libglusterfs.so.0(poll_iteration+0x1d) [0xadbf1d]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8049e70]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/lib/libc.so.6(__libc_start_main+0xe0) [0x58af70]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8049a41]
]0;root@dhcp217:/usr/local/var/log/glusterfs [root@dhcp217 glusterfs]# vi
!$
vi glusterfs.log
[?1049h[?1h=[1;26r[?12;25h[?12l[?25h[27m[m[H[2J[?25l[26;1H"glusterfs.log"
"glusterfs.log" 103L, 13037C[1;1H2007-06-30 04:29:08 E
[xlator.c:115:xlator_set_type] libglusterfs/xlator:
dlopen(/usr/local/ll[2;1Hib/glusterfs/1.3.0-pre5/xlator/performance/stat-prefetch.so):
/usr/local/lib/glusterfs/1.3.0--[3;1Hpre5/xlator/performance/stat-prefetch.so:
cannot open shared object file: No such file or dirr[4;1Hectory
2007-06-30 04:41:27 C [unify.c:4379:init] bricks: namespace option not
specified, Exiting
2007-06-30 04:41:27 C [common-utils.c:205:gf_print_trace] debug-backtrace:
Got signal (11), pp[7;1Hrinting backtrace
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[9;1Hglusterfs.so.0(gf_print_trace+0x2d) [0xef994d]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[0xc8d420]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[12;1Hglusterfs.so.0(default_notify+0x5b) [0xef7bbb]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[14;1Hglusterfs.so.0(default_notify+0x5b) [0xef7bbb]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[16;1Hglusterfs.so.0(default_notify+0x5b) [0xef7bbb]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[18;1Hglusterfs.so.0(xlator_tree_init+0x53) [0xef6c73]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8044[20;1Hd90d]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[22;1Hfuse.so.2 [0xe86bef]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[24;1Hfuse.so.2 [0xe867a3]
2007-06-30 04:41:27 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/lib[25;1H[1m[34m@
[1;1H[?12l[?25h[?25l[27m[m[H[2J[1;1H2007-07-01 02:52:06 C
[common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[2;1Hglusterfs.so.0(default_lookup+0x85) [0xada795]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8044[4;1Hd3d6]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[6;1Hfuse.so.2 [0x8ded61]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[8;1Hfuse.so.2 [0x8df7a3]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[10;1Hfuse.so.2(fuse_session_process+0x26) [0x8e0fa6]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8044[12;1Ha740]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[14;1Hglusterfs.so.0(transport_notify+0x37) [0xadbe47]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[16;1Hglusterfs.so.0(sys_epoll_iteration+0xd9)
[0xadc969]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[18;1Hglusterfs.so.0(poll_iteration+0x1d) [0xadbf1d]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8044[20;1H9e70]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/lib/libc.so.6(__ll[22;1Hibc_start_main+0xe0) [0x58af70]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[glusterfs] [0x8044[24;1H9a41]
[1m[34m~
[23;1H[?12l[?25h[?25l[1;25r[m[1;1H[21L[1;26r[1;1H2007-07-01
01:54:44 E [tcp-client.c:135:tcp_connect] device128-500GB-128IDEss: error:
not in [2;1Hprogress - trace: Network is unreachable
2007-07-01 01:54:45 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in [4;1Hprogress - trace: Network is
unreachable
2007-07-01 01:54:45 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in [6;1Hprogress - trace: Network is
unreachable
2007-07-01 01:54:46 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in [8;1Hprogress - trace: Network is
unreachable
2007-07-01 01:54:46 E [tcp-client.c:135:tcp_connect]
device128-500GB-128IDEss: error: not in [10;1Hprogress - trace: Network is
unreachable
2007-07-01 01:54:50 E [fuse-bridge.c:1910:fuse_transport_notify]
glusterfs-fuse: got GF_EVENTT[12;1H_POLLERR
2007-07-01 02:52:06 C [common-utils.c:205:gf_print_trace] debug-backtrace:
Got signal (11), pp[14;1Hrinting backtrace
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[16;1Hglusterfs.so.0(gf_print_trace+0x2d) [0xada94d]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
[0xf53420]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/gluu[19;1Hsterfs/1.3.0-pre5/xlator/cluster/afr.so [0xf91225]
2007-07-01 02:52:06 C [common-utils.c:207:gf_print_trace] debug-backtrace:
/usr/local/lib/libb[21;1Hglusterfs.so.0(default_lookup+0x85) [0xada795]
[?12l[?25h[22;1H[20;1H[18;1H[17;1H[15;1H[13;1H[11;1H[9;1H
[?25l[26;1H:[?12l[?25hq
t@dhcp217:/usr/local/var/log/glusterfs [
root@dhcp217 glusterfs]# ping device128
PING device128 (192.168.1.128) 56(84) bytes of data.
64 bytes from 192.168.1.128: icmp_seq=1 ttl=64 time=2.49 ms
64 bytes from 192.168.1.128: icmp_seq=2 ttl=64 time=0.354 ms
--- device128 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1002ms
rtt min/avg/max/mdev = 0.354/1.425/2.497/1.072 ms
]0;root@dhcp217:/usr/local/var/log/glusterfs [root@dhcp217 glusterfs]#
telnet device128 6996
Trying 192.168.1.128...
Connected to device128.
Escape character is '^]'.
^]
telnet> quit
Connection closed.
]0;root@dhcp217:/usr/local/var/log/glusterfs [root@dhcp217 glusterfs]#
Script done on Sun 01 Jul 2007 02:55:32 AM PDT
_________________________________________________________________
Like puzzles? Play free games & earn great prizes. Play Clink now.
http://club.live.com/clink.aspx?icid=clink_hotmailtextlink2
