Hi,
Kotresh workaround works for me. But before I tried it, I created some strace-logs for Florian.
setup: 2 VMs (192.168.222.120 master, 192.168.222.121 slave), both with a volume named vol with Ubuntu 16.04.3, glusterfs 3.13.2, rsync 3.1.1 .
Best regards,
Tino
root@master:~# cat /usr/bin/rsync
#!/bin/bash
strace -o /tmp/rsync.trace -ff /usr/bin/rsynco "$@"
One of the traces (looking all the same, execpt for memory addresses)
execve("/usr/bin/rsynco", ["/usr/bin/rsynco", "-aR0", "--inplace", "--files-from=-", "--super", "--stats", "--numeric-ids", "--no-implied-dirs", "--existing", "--xattrs", "--acls", "--ignore-missing-args", ".", "-e", "ssh -oPasswordAuthentication=no "..., "--compress", ...], [/* 7 vars */]) = 0
brk(NULL) = 0x563d3fc3e000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=24613, ...}) = 0
mmap(NULL, 24613, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7efdccfa5000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\20\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18624, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7efdccfa4000
mmap(NULL, 2113760, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7efdccb82000
mprotect(0x7efdccb86000, 2093056, PROT_NONE) = 0
mmap(0x7efdccd85000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7efdccd85000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\34\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=31232, ...}) = 0
mmap(NULL, 2126336, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7efdcc97a000
mprotect(0x7efdcc981000, 2093056, PROT_NONE) = 0
mmap(0x7efdccb80000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7efdccb80000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpopt.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360!\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=48240, ...}) = 0
mmap(NULL, 2143440, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7efdcc76e000
mprotect(0x7efdcc779000, 2093056, PROT_NONE) = 0
mmap(0x7efdcc978000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7efdcc978000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\t\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7efdccfa3000
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7efdcc3a4000
mprotect(0x7efdcc564000, 2097152, PROT_NONE) = 0
mmap(0x7efdcc764000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7efdcc764000
mmap(0x7efdcc76a000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7efdcc76a000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7efdccfa2000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7efdccfa1000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7efdccfa0000
arch_prctl(ARCH_SET_FS, 0x7efdccfa1700) = 0
mprotect(0x7efdcc764000, 16384, PROT_READ) = 0
mprotect(0x7efdcc978000, 4096, PROT_READ) = 0
mprotect(0x7efdccd85000, 4096, PROT_READ) = 0
mprotect(0x7efdccb80000, 4096, PROT_READ) = 0
mprotect(0x563d3e0c8000, 8192, PROT_READ) = 0
mprotect(0x7efdccfac000, 4096, PROT_READ) = 0
munmap(0x7efdccfa5000, 24613) = 0
rt_sigaction(SIGUSR1, {0x563d3de7c170, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigaction(SIGUSR2, {0x563d3de7ce10, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigaction(SIGCHLD, {0x563d3de7c0e0, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
geteuid() = 0
getegid() = 0
umask(0) = 022
umask(022) = 0
brk(NULL) = 0x563d3fc3e000
brk(0x563d3fc5f000) = 0x563d3fc5f000
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2981280, ...}) = 0
mmap(NULL, 2981280, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7efdcc0cc000
close(3) = 0
open("/usr/etc/popt", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/popt", O_RDONLY) = -1 ENOENT (No such file or directory)
stat("/etc/popt.d", 0x7ffff051e980) = -1 ENOENT (No such file or directory)
rt_sigaction(SIGINT, {0x563d3de6bac0, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigaction(SIGHUP, {0x563d3de6bac0, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigaction(SIGTERM, {0x563d3de6bac0, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [HUP INT USR1 USR2 TERM CHLD], NULL, 8) = 0
rt_sigaction(SIGPIPE, {SIG_IGN, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigaction(SIGXFSZ, {SIG_IGN, [], SA_RESTORER|SA_NOCLDSTOP, 0x7efdcc3d94b0}, NULL, 8) = 0
getcwd("(unreachable)/", 4095) = 15
lstat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
lstat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
openat(AT_FDCWD, "..", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
fstat(3, {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
fcntl(3, F_GETFL) = 0x8000 (flags O_RDONLY|O_LARGEFILE)
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
mmap(NULL, 135168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7efdccf7f000
getdents(3, /* 6 entries */, 131072) = 168
getdents(3, /* 0 entries */, 131072) = 0
lseek(3, 0, SEEK_SET) = 0
getdents(3, /* 6 entries */, 131072) = 168
newfstatat(3, "test", {st_mode=S_IFDIR|0755, st_size=4096, ...}, AT_SYMLINK_NOFOLLOW) = 0
newfstatat(3, ".trashcan", {st_mode=S_IFDIR|0755, st_size=4096, ...}, AT_SYMLINK_NOFOLLOW) = 0
getdents(3, /* 0 entries */, 131072) = 0
munmap(0x7efdccf7f000, 135168) = 0
close(3) = 0
write(2, "rsync: getcwd(): No such file or"..., 46) = 46
write(2, "\n", 1) = 1
rt_sigaction(SIGUSR1, {SIG_IGN, [], SA_RESTORER, 0x7efdcc3d94b0}, NULL, 8) = 0
rt_sigaction(SIGUSR2, {SIG_IGN, [], SA_RESTORER, 0x7efdcc3d94b0}, NULL, 8) = 0
write(2, "rsync error: errors selecting in"..., 96) = 96
write(2, "\n", 1) = 1
exit_group(3) = ?
+++ exited with 3 +++
Gesendet: Dienstag, 06. Februar 2018 um 10:44 Uhr
Von: "Kotresh Hiremath Ravishankar" <khiremat@xxxxxxxxxx>
An: "Florian Weimer" <fweimer@xxxxxxxxxx>
Cc: "Gluster Users" <gluster-users@xxxxxxxxxxx>
Betreff: Re: geo-replication command rsync returned with 3
Hi,
As a quick workaround for geo-replication to work. Please configure the following option.
gluster vol geo-replication <mastervol> <slavehost>::<slavevol> config access_mount true
The above option will not do the lazy umount and as a result, all the master and slave volume mountsmaintained by geo-replication can be accessed by others. It's also visible in df output.There might be cases where the mount points not get cleaned up when worker goes faulty and come back.These needs manual cleaning.
Thanks,Kotresh HR
On Tue, Feb 6, 2018 at 12:37 AM, Florian Weimer <fweimer@xxxxxxxxxx[mailto:fweimer@xxxxxxxxxx]> wrote:On 02/05/2018 01:33 PM, Florian Weimer wrote:
Do you have strace output going further back, at least to the proceeding getcwd call? It would be interesting to see which path the kernel reports, and if it starts with "(unreachable)".
I got the strace output now, but it very difficult to read (chdir in a multi-threaded process …).
My current inclination is to blame rsync because it does an unconditional getcwd during startup, which now fails if the current directory is unreachable.
Further references:
https://sourceware.org/ml/libc-alpha/2018-02/msg00152.html[https://sourceware.org/ml/libc-alpha/2018-02/msg00152.html]
https://bugzilla.redhat.com/show_bug.cgi?id=1542180
Andreas Schwab agrees that rsync is buggy:
https://sourceware.org/ml/libc-alpha/2018-02/msg00153.html[https://sourceware.org/ml/libc-alpha/2018-02/msg00153.html]
Thanks,
Florian
_______________________________________________
Gluster-users mailing list
Gluster-users@xxxxxxxxxxx[mailto:Gluster-users@xxxxxxxxxxx]
http://lists.gluster.org/mailman/listinfo/gluster-users
--
Thanks and Regards,Kotresh H R_______________________________________________ Gluster-users mailing list Gluster-users@xxxxxxxxxxx http://lists.gluster.org/mailman/listinfo/gluster-users[http://lists.gluster.org/mailman/listinfo/gluster-users]
_______________________________________________
Gluster-users mailing list
Gluster-users@xxxxxxxxxxx
http://lists.gluster.org/mailman/listinfo/gluster-users
