Hi,
With the new release, I ran into a bit of a problem. I have six servers running a unified AFR mounted by a single client and after upgrading, I maintain the mount with the client. I get the following error when running "df" against the mountpoint:
df: `/mnt/glusterfs': Invalid argument
Here is an strace:
execve("/usr/sbin/glusterfs", ["glusterfs", "-f", "/etc/glusterfs/glusterfs-client."..., "/mnt/glusterfs/"], [/* 15 vars */]) = 0
uname({sys="Linux", node="nat1621", ...}) = 0
brk(0) = 0x8054000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fc1000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=28812, ...}) = 0
mmap2(NULL, 28812, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7fb9000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libglusterfs.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\3601\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=58048, ...}) = 0
mmap2(NULL, 61216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7faa000
mmap2(0xb7fb8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7fb8000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libibverbs.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \34\0\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=27696, ...}) = 0
mmap2(NULL, 30700, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7fa2000
mmap2(0xb7fa9000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7fa9000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libfuse.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\00005\0\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=73704, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa1000
mmap2(NULL, 76644, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f8e000
mmap2(0xb7fa0000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11) = 0xb7fa0000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/tls/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\f\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=9592, ...}) = 0
mmap2(NULL, 12404, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f8a000
mmap2(0xb7f8c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb7f8c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/tls/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360G\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=85010, ...}) = 0
mmap2(NULL, 70104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f78000
mmap2(0xb7f86000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7f86000
mmap2(0xb7f88000, 4568, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f88000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/tls/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240O\1"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=1241392, ...}) = 0
mmap2(NULL, 1251484, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e46000
mmap2(0xb7f6e000, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x127) = 0xb7f6e000
mmap2(0xb7f75000, 10396, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f75000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/libsysfs.so.2", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\37\0"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=37496, ...}) = 0
mmap2(NULL, 40508, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e3c000
mmap2(0xb7e45000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8) = 0xb7e45000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/tls/librt.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\35\0\000"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0644, st_size=26516, ...}) = 0
mmap2(NULL, 29264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e34000
mmap2(0xb7e3a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5) = 0xb7e3a000
close(3) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e33000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e32000
mprotect(0xb7f6e000, 20480, PROT_READ) = 0
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7e326c0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
munmap(0xb7fb9000, 28812) = 0
set_tid_address(0xb7e32708) = 31997
rt_sigaction(SIGRTMIN, {0xb7f7c450, [], SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0xb7f7c3c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="nat1621", ...}) = 0
setrlimit(RLIMIT_CORE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0
setrlimit(RLIMIT_NOFILE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = -1 EPERM (Operation not permitted)
brk(0) = 0x8054000
brk(0x8075000) = 0x8075000
open("/var/log/glusterfs/glusterfs.log", O_WRONLY|O_APPEND|O_CREAT|O_LARGEFILE, 0666) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fc0000
fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
_llseek(3, 29947462, [29947462], SEEK_SET) = 0
open("/etc/glusterfs/glusterfs-client.vol", O_RDONLY|O_LARGEFILE) = 4
rt_sigaction(SIGPIPE, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGSEGV, {0xb7fb36b0, [SEGV], SA_RESTART}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGABRT, {0xb7fb36b0, [ABRT], SA_RESTART}, {SIG_DFL}, 8) = 0
socketpair(PF_FILE, SOCK_STREAM, 0, [5, 6]) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0xb7e32708) = 31998
close(5) = 0
recvmsg(6, {msg_name(0)=NULL, msg_iov(1)=[{"\0", 1}], msg_controllen=16, {cmsg_len=16, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {5}}, msg_flags=0}, 0) = 1
--- SIGCHLD (Child exited) @ 0 (0) ---
close(6) = 0
waitpid(31998, NULL, 0) = 31998
getuid32() = 0
rt_sigaction(SIGHUP, NULL, {SIG_DFL}, 8) = 0
rt_sigaction(SIGHUP, {0xb7f9ca80, [], 0}, NULL, 8) = 0
rt_sigaction(SIGINT, NULL, {SIG_DFL}, 8) = 0
rt_sigaction(SIGINT, {0xb7f9ca80, [], 0}, NULL, 8) = 0
rt_sigaction(SIGTERM, NULL, {SIG_DFL}, 8) = 0
rt_sigaction(SIGTERM, {0xb7f9ca80, [], 0}, NULL, 8) = 0
rt_sigaction(SIGPIPE, NULL, {SIG_IGN}, 8) = 0
epoll_create(1024) = 6
epoll_ctl(6, EPOLL_CTL_ADD, 5, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=134562696, u64=134562696}}) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0xb7e32708) = 31999
exit_group(0) = ?
Process 31997 detached
---------------------------------------
and here is a copy of my client and server config:
----------------------------------------
client.vol
volume client1a
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.1
option remote-port 6996
option remote-subvolume iothreads1
end-volume
volume client1b
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.1
option remote-port 6996
option remote-subvolume iothreads2
end-volume
volume client1c
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.1
option remote-port 6996
option remote-subvolume iothreads3
end-volume
volume client1d
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.1
option remote-port 6996
option remote-subvolume iothreads4
end-volume
volume client2a
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.2
option remote-port 6996
option remote-subvolume iothreads1
end-volume
volume client2b
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.2
option remote-port 6996
option remote-subvolume iothreads2
end-volume
volume client2c
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.2
option remote-port 6996
option remote-subvolume iothreads3
end-volume
volume client2d
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.2
option remote-port 6996
option remote-subvolume iothreads4
end-volume
volume client3a
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.3
option remote-port 6996
option remote-subvolume iothreads1
end-volume
volume client3b
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.3
option remote-port 6996
option remote-subvolume iothreads2
end-volume
volume client3c
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.3
option remote-port 6996
option remote-subvolume iothreads3
end-volume
volume client3d
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.3
option remote-port 6996
option remote-subvolume iothreads4
end-volume
volume client4a
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.4
option remote-port 6996
option remote-subvolume iothreads1
end-volume
volume client4b
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.4
option remote-port 6996
option remote-subvolume iothreads2
end-volume
volume client4c
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.4
option remote-port 6996
option remote-subvolume iothreads3
end-volume
volume client4d
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.4
option remote-port 6996
option remote-subvolume iothreads4
end-volume
volume client5a
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.5
option remote-port 6996
option remote-subvolume iothreads1
end-volume
volume client5b
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.5
option remote-port 6996
option remote-subvolume iothreads2
end-volume
volume client5c
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.5
option remote-port 6996
option remote-subvolume iothreads3
end-volume
volume client5d
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.5
option remote-port 6996
option remote-subvolume iothreads4
end-volume
volume client6a
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.6
option remote-port 6996
option remote-subvolume iothreads1
end-volume
volume client6b
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.6
option remote-port 6996
option remote-subvolume iothreads2
end-volume
volume client6c
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.6
option remote-port 6996
option remote-subvolume iothreads3
end-volume
volume client6d
type protocol/client
option transport-type tcp/client
option remote-host 10.100.0.6
option remote-port 6996
option remote-subvolume iothreads4
end-volume
volume afr1
type cluster/afr
subvolumes client1a client2a
option replicate *:2
end-volume
volume afr2
type cluster/afr
subvolumes client1b client2b
option replicate *:2
end-volume
volume afr3
type cluster/afr
subvolumes client1c client2c
option replicate *:2
end-volume
volume afr4
type cluster/afr
subvolumes client1d client2d
option replicate *:2
end-volume
volume afr5
type cluster/afr
subvolumes client3a client4a
option replicate *:2
end-volume
volume afr6
type cluster/afr
subvolumes client3b client4b
option replicate *:2
end-volume
volume afr7
type cluster/afr
subvolumes client3c client4c
option replicate *:2
end-volume
volume afr8
type cluster/afr
subvolumes client3d client4d
option replicate *:2
end-volume
volume afr9
type cluster/afr
subvolumes client5a client6a
option replicate *:2
end-volume
volume afr10
type cluster/afr
subvolumes client5b client6b
option replicate *:2
end-volume
volume afr11
type cluster/afr
subvolumes client5c client6c
option replicate *:2
end-volume
volume afr12
type cluster/afr
subvolumes client5d client6d
option replicate *:2
end-volume
volume unify
type cluster/unify
subvolumes afr1 afr2 afr3 afr4 afr5 afr6 afr7 afr8 afr9 afr10 afr11 afr12
option scheduler alu
option alu.limits.min-free-disk 60G
option alu.limits.max-open-files 10000
option alu.order disk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
option alu.disk-usage.entry-threshold 2GB
option alu.disk-usage.exit-threshold 64MB
option alu.open-files-usage.entry-threshold 1024
option alu.open-files-usage.exit-threshold 32
option alu.read-usage.entry-threshold 20%
option alu.read-usage.exit-threshold 4%
option alu.write-usage.entry-threshold 20%
option alu.write-usage.exit-threshold 4%
option alu.stat-refresh.interval 10sec
option alu.stat-refresh.num-file-create 10
option readdir-force-success on
end-volume
#volume writebehind
# type performance/write-behind
# option aggregate-size 131072
# subvolumes unify
#end-volume
volume readahead
type performance/read-ahead
option page-size 131072
option page-count 16
subvolumes unify
end-volume
volume statprefetch
type performance/stat-prefetch
option cache-seconds 2
subvolumes readahead
end-volume
---------------------
server.vol
volume volume1
type storage/posix
option directory /storage/vol1
end-volume
volume readahead1
type performance/read-ahead
option page-size 131072
option page-count 16
subvolumes volume1
end-volume
volume iothreads1
type performance/io-threads
option thread-count 8
subvolumes readahead1
end-volume
volume volume2
type storage/posix
option directory /storage/vol2
end-volume
volume readahead2
type performance/read-ahead
option page-size 131072
option page-count 16
subvolumes volume2
end-volume
volume iothreads2
type performance/io-threads
option thread-count 8
subvolumes readahead2
end-volume
volume volume3
type storage/posix
option directory /storage/vol3
end-volume
volume readahead3
type performance/read-ahead
option page-size 131072
option page-count 16
subvolumes volume3
end-volume
volume iothreads3
type performance/io-threads
option thread-count 8
subvolumes readahead3
end-volume
volume volume4
type storage/posix
option directory /storage/vol4
end-volume
volume readahead4
type performance/read-ahead
option page-size 131072
option page-count 16
subvolumes volume4
end-volume
volume iothreads4
type performance/io-threads
option thread-count 8
subvolumes readahead4
end-volume
volume server
type protocol/server
option transport-type tcp/server
subvolumes iothreads1 iothreads2 iothreads3 iothreads4
option auth.ip.iothreads1.allow 10.100.0.*
option auth.ip.iothreads2.allow 10.100.0.*
option auth.ip.iothreads3.allow 10.100.0.*
option auth.ip.iothreads4.allow 10.100.0.*
end-volume
-------------------------------------
Let me know if you need more information.
Thanks,
Majied Najjar
