Re: client cannot maintain mount of unified AFR

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Also,

This happened when the first client in the client config rebooted.  Normally, the second client in the afr group would have picked up the slack, but instead I was getting connection refused from the client.  I am assuming this is a locking issue?

Majied Najjar

On Wed, 27 Jun 2007 12:01:35 -0400
Majied Najjar <majied.najjar@xxxxxxxxxxxxxxx> wrote:

> Hi,
> 
> With the new release, I ran into a bit of a problem.  I have six servers running a unified AFR mounted by a single client and after upgrading, I maintain the mount with the client.  I get the following error when running "df" against the mountpoint:
> 
> df: `/mnt/glusterfs': Invalid argument
> 
> Here is an strace:
> 
> execve("/usr/sbin/glusterfs", ["glusterfs", "-f", "/etc/glusterfs/glusterfs-client."..., "/mnt/glusterfs/"], [/* 15 vars */]) = 0
> uname({sys="Linux", node="nat1621", ...}) = 0
> brk(0)                                  = 0x8054000
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fc1000
> access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> open("/etc/ld.so.cache", O_RDONLY)      = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=28812, ...}) = 0
> mmap2(NULL, 28812, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7fb9000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/usr/lib/libglusterfs.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\3601\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=58048, ...}) = 0
> mmap2(NULL, 61216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7faa000
> mmap2(0xb7fb8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7fb8000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/usr/lib/libibverbs.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \34\0\000"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=27696, ...}) = 0
> mmap2(NULL, 30700, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7fa2000
> mmap2(0xb7fa9000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7fa9000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/usr/lib/libfuse.so.2", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\00005\0\000"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=73704, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fa1000
> mmap2(NULL, 76644, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f8e000
> mmap2(0xb7fa0000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11) = 0xb7fa0000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/lib/tls/libdl.so.2", O_RDONLY)   = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\f\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=9592, ...}) = 0
> mmap2(NULL, 12404, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f8a000
> mmap2(0xb7f8c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb7f8c000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/lib/tls/libpthread.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360G\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=85010, ...}) = 0
> mmap2(NULL, 70104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7f78000
> mmap2(0xb7f86000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd) = 0xb7f86000
> mmap2(0xb7f88000, 4568, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f88000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/lib/tls/libc.so.6", O_RDONLY)    = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\240O\1"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=1241392, ...}) = 0
> mmap2(NULL, 1251484, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e46000
> mmap2(0xb7f6e000, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x127) = 0xb7f6e000
> mmap2(0xb7f75000, 10396, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7f75000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/lib/libsysfs.so.2", O_RDONLY)    = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\37\0"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=37496, ...}) = 0
> mmap2(NULL, 40508, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e3c000
> mmap2(0xb7e45000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8) = 0xb7e45000
> close(3)                                = 0
> access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
> open("/lib/tls/librt.so.1", O_RDONLY)   = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\35\0\000"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0644, st_size=26516, ...}) = 0
> mmap2(NULL, 29264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7e34000
> mmap2(0xb7e3a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5) = 0xb7e3a000
> close(3)                                = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e33000
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7e32000
> mprotect(0xb7f6e000, 20480, PROT_READ)  = 0
> set_thread_area({entry_number:-1 -> 6, base_addr:0xb7e326c0, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> munmap(0xb7fb9000, 28812)               = 0
> set_tid_address(0xb7e32708)             = 31997
> rt_sigaction(SIGRTMIN, {0xb7f7c450, [], SA_SIGINFO}, NULL, 8) = 0
> rt_sigaction(SIGRT_1, {0xb7f7c3c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> uname({sys="Linux", node="nat1621", ...}) = 0
> setrlimit(RLIMIT_CORE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = 0
> setrlimit(RLIMIT_NOFILE, {rlim_cur=RLIM_INFINITY, rlim_max=RLIM_INFINITY}) = -1 EPERM (Operation not permitted)
> brk(0)                                  = 0x8054000
> brk(0x8075000)                          = 0x8075000
> open("/var/log/glusterfs/glusterfs.log", O_WRONLY|O_APPEND|O_CREAT|O_LARGEFILE, 0666) = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7fc0000
> fstat64(3, {st_mode=S_IFREG|0644, st_size=29947462, ...}) = 0
> _llseek(3, 29947462, [29947462], SEEK_SET) = 0
> open("/etc/glusterfs/glusterfs-client.vol", O_RDONLY|O_LARGEFILE) = 4
> rt_sigaction(SIGPIPE, {SIG_IGN}, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGSEGV, {0xb7fb36b0, [SEGV], SA_RESTART}, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGABRT, {0xb7fb36b0, [ABRT], SA_RESTART}, {SIG_DFL}, 8) = 0
> socketpair(PF_FILE, SOCK_STREAM, 0, [5, 6]) = 0
> clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0xb7e32708) = 31998
> close(5)                                = 0
> recvmsg(6, {msg_name(0)=NULL, msg_iov(1)=[{"\0", 1}], msg_controllen=16, {cmsg_len=16, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {5}}, msg_flags=0}, 0) = 1
> --- SIGCHLD (Child exited) @ 0 (0) ---
> close(6)                                = 0
> waitpid(31998, NULL, 0)                 = 31998
> getuid32()                              = 0
> rt_sigaction(SIGHUP, NULL, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGHUP, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> rt_sigaction(SIGINT, NULL, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGINT, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> rt_sigaction(SIGTERM, NULL, {SIG_DFL}, 8) = 0
> rt_sigaction(SIGTERM, {0xb7f9ca80, [], 0}, NULL, 8) = 0
> rt_sigaction(SIGPIPE, NULL, {SIG_IGN}, 8) = 0
> epoll_create(1024)                      = 6
> epoll_ctl(6, EPOLL_CTL_ADD, 5, {EPOLLIN|EPOLLPRI|EPOLLERR|EPOLLHUP, {u32=134562696, u64=134562696}}) = 0
> clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0xb7e32708) = 31999
> exit_group(0)                           = ?
> Process 31997 detached
> 
> ---------------------------------------
> 
> and here is a copy of my client and server config:
> 
> ----------------------------------------
> 
> client.vol
> 
> volume client1a
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.1
>   option remote-port 6996
>   option remote-subvolume iothreads1
> end-volume
> 
> volume client1b
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.1
>   option remote-port 6996
>   option remote-subvolume iothreads2
> end-volume
> 
> volume client1c
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.1
>   option remote-port 6996
>   option remote-subvolume iothreads3
> end-volume
> 
> volume client1d
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.1
>   option remote-port 6996
>   option remote-subvolume iothreads4
> end-volume
> 
> volume client2a
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.2
>   option remote-port 6996
>   option remote-subvolume iothreads1
> end-volume
> 
> volume client2b
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.2
>   option remote-port 6996
>   option remote-subvolume iothreads2
> end-volume
> 
> volume client2c
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.2
>   option remote-port 6996
>   option remote-subvolume iothreads3
> end-volume
> 
> volume client2d
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.2
>   option remote-port 6996
>   option remote-subvolume iothreads4
> end-volume
> 
> volume client3a
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.3
>   option remote-port 6996
>   option remote-subvolume iothreads1
> end-volume
> 
> volume client3b
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.3
>   option remote-port 6996
>   option remote-subvolume iothreads2
> end-volume
> 
> volume client3c
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.3
>   option remote-port 6996
>   option remote-subvolume iothreads3
> end-volume
> 
> volume client3d
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.3
>   option remote-port 6996
>   option remote-subvolume iothreads4
> end-volume
> 
> volume client4a
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.4
>   option remote-port 6996
>   option remote-subvolume iothreads1
> end-volume
> 
> volume client4b
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.4
>   option remote-port 6996
>   option remote-subvolume iothreads2
> end-volume
> 
> volume client4c
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.4
>   option remote-port 6996
>   option remote-subvolume iothreads3
> end-volume
> 
> volume client4d
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.4
>   option remote-port 6996
>   option remote-subvolume iothreads4
> end-volume
> 
> volume client5a
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.5
>   option remote-port 6996
>   option remote-subvolume iothreads1
> end-volume
> 
> volume client5b
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.5
>   option remote-port 6996
>   option remote-subvolume iothreads2
> end-volume
> 
> volume client5c
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.5
>   option remote-port 6996
>   option remote-subvolume iothreads3
> end-volume
> 
> volume client5d
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.5
>   option remote-port 6996
>   option remote-subvolume iothreads4
> end-volume
> 
> volume client6a
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.6
>   option remote-port 6996
>   option remote-subvolume iothreads1
> end-volume
> 
> volume client6b
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.6
>   option remote-port 6996
>   option remote-subvolume iothreads2
> end-volume
> 
> volume client6c
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.6
>   option remote-port 6996
>   option remote-subvolume iothreads3
> end-volume
> 
> volume client6d
>   type protocol/client
>   option transport-type tcp/client
>   option remote-host 10.100.0.6
>   option remote-port 6996
>   option remote-subvolume iothreads4
> end-volume
> 
> volume afr1
>   type cluster/afr
>   subvolumes client1a client2a
>   option replicate *:2
> end-volume
> 
> volume afr2
>   type cluster/afr
>   subvolumes client1b client2b
>   option replicate *:2
> end-volume
> 
> volume afr3
>   type cluster/afr
>   subvolumes client1c client2c
>   option replicate *:2
> end-volume
> 
> volume afr4
>   type cluster/afr
>   subvolumes client1d client2d
>   option replicate *:2
> end-volume
> 
> volume afr5
>   type cluster/afr
>   subvolumes client3a client4a
>   option replicate *:2
> end-volume
> 
> volume afr6
>   type cluster/afr
>   subvolumes client3b client4b
>   option replicate *:2
> end-volume
> 
> volume afr7
>   type cluster/afr
>   subvolumes client3c client4c
>   option replicate *:2
> end-volume
> 
> volume afr8
>   type cluster/afr
>   subvolumes client3d client4d
>   option replicate *:2
> end-volume
> 
> volume afr9
>   type cluster/afr
>   subvolumes client5a client6a
>   option replicate *:2
> end-volume
> 
> volume afr10
>   type cluster/afr
>   subvolumes client5b client6b
>   option replicate *:2
> end-volume
> 
> volume afr11
>   type cluster/afr
>   subvolumes client5c client6c
>   option replicate *:2
> end-volume
> 
> volume afr12
>   type cluster/afr
>   subvolumes client5d client6d
>   option replicate *:2
> end-volume
> 
> volume unify
>   type cluster/unify
>   subvolumes afr1 afr2 afr3 afr4 afr5 afr6 afr7 afr8 afr9 afr10 afr11 afr12
>   option scheduler alu
>   option alu.limits.min-free-disk 60G
>   option alu.limits.max-open-files 10000
>   option alu.order disk-usage:read-usage:write-usage:open-files-usage:disk-speed-usage
>   option alu.disk-usage.entry-threshold 2GB
>   option alu.disk-usage.exit-threshold 64MB
>   option alu.open-files-usage.entry-threshold 1024
>   option alu.open-files-usage.exit-threshold 32
>   option alu.read-usage.entry-threshold 20%
>   option alu.read-usage.exit-threshold 4%
>   option alu.write-usage.entry-threshold 20%
>   option alu.write-usage.exit-threshold 4%
>   option alu.stat-refresh.interval 10sec
>   option alu.stat-refresh.num-file-create 10
>   option readdir-force-success on
> end-volume
> 
> #volume writebehind
> #  type performance/write-behind
> #  option aggregate-size 131072
> #  subvolumes unify
> #end-volume
> 
> volume readahead
>   type performance/read-ahead
>   option page-size 131072
>   option page-count 16
>   subvolumes unify
> end-volume
> 
> volume statprefetch
>   type performance/stat-prefetch
>   option cache-seconds 2
>   subvolumes readahead
> end-volume
> 
> ---------------------
> 
> server.vol
> 
> volume volume1
>   type storage/posix
>   option directory /storage/vol1
> end-volume
> 
> volume readahead1
>   type performance/read-ahead
>   option page-size 131072
>   option page-count 16
>   subvolumes volume1
> end-volume
> 
> volume iothreads1
>   type performance/io-threads
>   option thread-count 8
>   subvolumes readahead1
> end-volume
> 
> volume volume2
>   type storage/posix
>   option directory /storage/vol2
> end-volume
> 
> volume readahead2
>   type performance/read-ahead
>   option page-size 131072
>   option page-count 16
>   subvolumes volume2
> end-volume
> 
> volume iothreads2
>   type performance/io-threads
>   option thread-count 8
>   subvolumes readahead2
> end-volume
> 
> volume volume3
>   type storage/posix
>   option directory /storage/vol3
> end-volume
> 
> volume readahead3
>   type performance/read-ahead
>   option page-size 131072
>   option page-count 16
>   subvolumes volume3
> end-volume
> 
> volume iothreads3
>   type performance/io-threads
>   option thread-count 8
>   subvolumes readahead3
> end-volume
> 
> volume volume4
>   type storage/posix
>   option directory /storage/vol4
> end-volume
> 
> volume readahead4
>   type performance/read-ahead
>   option page-size 131072
>   option page-count 16
>   subvolumes volume4
> end-volume
> 
> volume iothreads4
>   type performance/io-threads
>   option thread-count 8
>   subvolumes readahead4
> end-volume
> 
> volume server
>   type protocol/server
>   option transport-type tcp/server
>   subvolumes iothreads1 iothreads2 iothreads3 iothreads4
>   option auth.ip.iothreads1.allow 10.100.0.*
>   option auth.ip.iothreads2.allow 10.100.0.*
>   option auth.ip.iothreads3.allow 10.100.0.*
>   option auth.ip.iothreads4.allow 10.100.0.*
> end-volume
> -------------------------------------
> 
> Let me know if you need more information.
> 
> Thanks,
> Majied Najjar




[Index of Archives]     [Gluster Users]     [Ceph Users]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Security]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [eCos]     [Asterisk Internet PBX]     [Linux API]

  Powered by Linux