Also,
This happened when the first client in the client config rebooted. Normally, the second client in the afr group would have picked up the slack, but instead I was getting connection refused from the client. I am assuming this is a locking issue?
Majied Najjar
On Wed, 27 Jun 2007 12:01:35 -0400
Majied Najjar <majied.najjar@xxxxxxxxxxxxxxx> wrote:
> Hi,
>
> With the new release, I ran into a bit of a problem. I have six servers running a unified AFR mounted by a single client and after upgrading, I maintain the mount with the client. I get the following error when running "df" against the mountpoint:
>
> df: `/mnt/glusterfs': Invalid argument
>
> Here is an strace:
>
> execve("/usr/sbin/glusterfs", ["glusterfs", "-f", "/etc/glusterfs/glusterfs-client."..., "/mnt/glusterfs/"], [/* 15 vars */]) = 0
> uname({sys="Linux", node="nat1621", ...}) = 0
> brk(0) = 0x8054000
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fc1000
> access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
> open("/etc/ld.so.cache", O_RDONLY) = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=28812, ...}) = 0
> mmap2(NULL, 28812, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7fb9000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/usr/lib/libglusterfs.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\3601\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=58048, ...}) = 0
> mmap2(NULL, 61216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7faa000
> mmap2(0xb7fb8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7fb8000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/usr/lib/libibverbs.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \34\0\000"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=27696, ...}) = 0
> mmap2(NULL, 30700, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7fa2000
> mmap2(0xb7fa9000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7fa9000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/usr/lib/libfuse.so.2", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\00005\0\000"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=73704, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa1000
> mmap2(NULL, 76644, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f8e000
> mmap2(0xb7fa0000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11) = 0xb7fa0000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/lib/tls/libdl.so.2", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\f\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=9592, ...}) = 0
> mmap2(NULL, 12404, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f8a000
> mmap2(0xb7f8c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb7f8c000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/lib/tls/libpthread.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360G\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=85010, ...}) = 0
> mmap2(NULL, 70104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f78000
> mmap2(0xb7f86000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7f86000
> mmap2(0xb7f88000, 4568, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f88000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/lib/tls/libc.so.6", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240O\1"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=1241392, ...}) = 0
> mmap2(NULL, 1251484, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e46000
> mmap2(0xb7f6e000, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x127) = 0xb7f6e000
> mmap2(0xb7f75000, 10396, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f75000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/lib/libsysfs.so.2", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\37\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=37496, ...}) = 0
> mmap2(NULL, 40508, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e3c000
> mmap2(0xb7e45000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8) = 0xb7e45000
> close(3) = 0
> access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
> open("/lib/tls/librt.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\35\0\000"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=26516, ...}) = 0
> mmap2(NULL, 29264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e34000
> mmap2(0xb7e3a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5) = 0xb7e3a000
> close(3) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e33000
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e32000
> mprotect(0xb7f6e000, 20480, PROT_READ) = 0
> set_thread_area({entry_number:-1 -> 6, base_addr:0xb7e326c0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> munmap(0xb7fb9000, 28812) = 0
> set_tid_address(0xb7e32708) = 31997
> rt_sigaction(SIGRTMIN, {0xb7f7c450, [], SA_SIGINFO}, NULL, 8) = 0
> rt_sigaction(SIGRT_1, {0xb7f7c3c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> uname({sys="Linux", node="nat1621", ...}) = 0
> setrlimit(RLIMIT_CORE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0
> setrlimit(RLIMIT_NOFILE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = -1 EPERM (Operation not permitted)
> brk(0) = 0x8054000
> brk(0x8075000) = 0x8075000
> open("/var/log/glusterfs/glusterfs.log", O_WRONLY|O_APPEND|O_CREAT|O_LARGEFILE, 0666) = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fc0000
> fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
> _llseek(3, 29947462, [29947462], SEEK_SET) = 0
> open("/etc/glusterfs/glusterfs-client.vol", O_RDONLY|O_LARGEFILE) = 4
> rt_sigaction(SIGPIPE, {SIG_IGN}, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGSEGV, {0xb7fb36b0, [SEGV], SA_RESTART}, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGABRT, {0xb7fb36b0, [ABRT], SA_RESTART}, {SIG_DFL}, 8) = 0
> socketpair(PF_FILE, SOCK_STREAM, 0, [5, 6]) = 0
> clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0xb7e32708) = 31998
> close(5) = 0
> recvmsg(6, {msg_name(0)=NULL, msg_iov(1)=[{"\0", 1}], msg_controllen=16, {cmsg_len=16, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {5}}, msg_flags=0}, 0) = 1
> --- SIGCHLD (Child exited) @ 0 (0) ---
> close(6) = 0
> waitpid(31998, NULL, 0) = 31998
> getuid32() = 0
> rt_sigaction(SIGHUP, NULL, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGHUP, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> rt_sigaction(SIGINT, NULL, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGINT, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> rt_sigaction(SIGTERM, NULL, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGTERM, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> rt_sigaction(SIGPIPE, NULL, {SIG_IGN}, 8) = 0
> epoll_create(1024) = 6
> epoll_ctl(6, EPOLL_CTL_ADD, 5, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=134562696, u64=134562696}}) = 0
> clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0xb7e32708) = 31999
> exit_group(0) = ?
> Process 31997 detached
>
> ---------------------------------------
>
> and here is a copy of my client and server config:
>
> ----------------------------------------
>
> client.vol
>
> volume client1a
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.1
> option remote-port 6996
> option remote-subvolume iothreads1
> end-volume
>
> volume client1b
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.1
> option remote-port 6996
> option remote-subvolume iothreads2
> end-volume
>
> volume client1c
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.1
> option remote-port 6996
> option remote-subvolume iothreads3
> end-volume
>
> volume client1d
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.1
> option remote-port 6996
> option remote-subvolume iothreads4
> end-volume
>
> volume client2a
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.2
> option remote-port 6996
> option remote-subvolume iothreads1
> end-volume
>
> volume client2b
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.2
> option remote-port 6996
> option remote-subvolume iothreads2
> end-volume
>
> volume client2c
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.2
> option remote-port 6996
> option remote-subvolume iothreads3
> end-volume
>
> volume client2d
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.2
> option remote-port 6996
> option remote-subvolume iothreads4
> end-volume
>
> volume client3a
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.3
> option remote-port 6996
> option remote-subvolume iothreads1
> end-volume
>
> volume client3b
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.3
> option remote-port 6996
> option remote-subvolume iothreads2
> end-volume
>
> volume client3c
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.3
> option remote-port 6996
> option remote-subvolume iothreads3
> end-volume
>
> volume client3d
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.3
> option remote-port 6996
> option remote-subvolume iothreads4
> end-volume
>
> volume client4a
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.4
> option remote-port 6996
> option remote-subvolume iothreads1
> end-volume
>
> volume client4b
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.4
> option remote-port 6996
> option remote-subvolume iothreads2
> end-volume
>
> volume client4c
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.4
> option remote-port 6996
> option remote-subvolume iothreads3
> end-volume
>
> volume client4d
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.4
> option remote-port 6996
> option remote-subvolume iothreads4
> end-volume
>
> volume client5a
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.5
> option remote-port 6996
> option remote-subvolume iothreads1
> end-volume
>
> volume client5b
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.5
> option remote-port 6996
> option remote-subvolume iothreads2
> end-volume
>
> volume client5c
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.5
> option remote-port 6996
> option remote-subvolume iothreads3
> end-volume
>
> volume client5d
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.5
> option remote-port 6996
> option remote-subvolume iothreads4
> end-volume
>
> volume client6a
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.6
> option remote-port 6996
> option remote-subvolume iothreads1
> end-volume
>
> volume client6b
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.6
> option remote-port 6996
> option remote-subvolume iothreads2
> end-volume
>
> volume client6c
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.6
> option remote-port 6996
> option remote-subvolume iothreads3
> end-volume
>
> volume client6d
> type protocol/client
> option transport-type tcp/client
> option remote-host 10.100.0.6
> option remote-port 6996
> option remote-subvolume iothreads4
> end-volume
>
> volume afr1
> type cluster/afr
> subvolumes client1a client2a
> option replicate *:2
> end-volume
>
> volume afr2
> type cluster/afr
> subvolumes client1b client2b
> option replicate *:2
> end-volume
>
> volume afr3
> type cluster/afr
> subvolumes client1c client2c
> option replicate *:2
> end-volume
>
> volume afr4
> type cluster/afr
> subvolumes client1d client2d
> option replicate *:2
> end-volume
>
> volume afr5
> type cluster/afr
> subvolumes client3a client4a
> option replicate *:2
> end-volume
>
> volume afr6
> type cluster/afr
> subvolumes client3b client4b
> option replicate *:2
> end-volume
>
> volume afr7
> type cluster/afr
> subvolumes client3c client4c
> option replicate *:2
> end-volume
>
> volume afr8
> type cluster/afr
> subvolumes client3d client4d
> option replicate *:2
> end-volume
>
> volume afr9
> type cluster/afr
> subvolumes client5a client6a
> option replicate *:2
> end-volume
>
> volume afr10
> type cluster/afr
> subvolumes client5b client6b
> option replicate *:2
> end-volume
>
> volume afr11
> type cluster/afr
> subvolumes client5c client6c
> option replicate *:2
> end-volume
>
> volume afr12
> type cluster/afr
> subvolumes client5d client6d
> option replicate *:2
> end-volume
>
> volume unify
> type cluster/unify
> subvolumes afr1 afr2 afr3 afr4 afr5 afr6 afr7 afr8 afr9 afr10 afr11 afr12
> option scheduler alu
> option alu.limits.min-free-disk 60G
> option alu.limits.max-open-files 10000
> option alu.order disk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
> option alu.disk-usage.entry-threshold 2GB
> option alu.disk-usage.exit-threshold 64MB
> option alu.open-files-usage.entry-threshold 1024
> option alu.open-files-usage.exit-threshold 32
> option alu.read-usage.entry-threshold 20%
> option alu.read-usage.exit-threshold 4%
> option alu.write-usage.entry-threshold 20%
> option alu.write-usage.exit-threshold 4%
> option alu.stat-refresh.interval 10sec
> option alu.stat-refresh.num-file-create 10
> option readdir-force-success on
> end-volume
>
> #volume writebehind
> # type performance/write-behind
> # option aggregate-size 131072
> # subvolumes unify
> #end-volume
>
> volume readahead
> type performance/read-ahead
> option page-size 131072
> option page-count 16
> subvolumes unify
> end-volume
>
> volume statprefetch
> type performance/stat-prefetch
> option cache-seconds 2
> subvolumes readahead
> end-volume
>
> ---------------------
>
> server.vol
>
> volume volume1
> type storage/posix
> option directory /storage/vol1
> end-volume
>
> volume readahead1
> type performance/read-ahead
> option page-size 131072
> option page-count 16
> subvolumes volume1
> end-volume
>
> volume iothreads1
> type performance/io-threads
> option thread-count 8
> subvolumes readahead1
> end-volume
>
> volume volume2
> type storage/posix
> option directory /storage/vol2
> end-volume
>
> volume readahead2
> type performance/read-ahead
> option page-size 131072
> option page-count 16
> subvolumes volume2
> end-volume
>
> volume iothreads2
> type performance/io-threads
> option thread-count 8
> subvolumes readahead2
> end-volume
>
> volume volume3
> type storage/posix
> option directory /storage/vol3
> end-volume
>
> volume readahead3
> type performance/read-ahead
> option page-size 131072
> option page-count 16
> subvolumes volume3
> end-volume
>
> volume iothreads3
> type performance/io-threads
> option thread-count 8
> subvolumes readahead3
> end-volume
>
> volume volume4
> type storage/posix
> option directory /storage/vol4
> end-volume
>
> volume readahead4
> type performance/read-ahead
> option page-size 131072
> option page-count 16
> subvolumes volume4
> end-volume
>
> volume iothreads4
> type performance/io-threads
> option thread-count 8
> subvolumes readahead4
> end-volume
>
> volume server
> type protocol/server
> option transport-type tcp/server
> subvolumes iothreads1 iothreads2 iothreads3 iothreads4
> option auth.ip.iothreads1.allow 10.100.0.*
> option auth.ip.iothreads2.allow 10.100.0.*
> option auth.ip.iothreads3.allow 10.100.0.*
> option auth.ip.iothreads4.allow 10.100.0.*
> end-volume
> -------------------------------------
>
> Let me know if you need more information.
>
> Thanks,
> Majied Najjar
