Re: rbd command hangs

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Hi Ilya,


thank you very much for your prompt response,


Any rbd command variation is affected (mapping device included)

We are using a physical machine (no container involved)


Below is the output of the running strace as suggested:


nid001388:/usr/src/linux # strace -f rbd -n client.noir -o noudev map noir-nvme-meta/nid001388
execve("/usr/bin/rbd", ["rbd", "-n", "client.noir", "-o", "noudev", "map", "noir-nvme-meta/nid001388"], 0x7ffc33caafe8 /* 63 vars */) = 0
brk(NULL)                               = 0x55ee2015c000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/tls/x86_64/x86_64/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/tls/x86_64/x86_64", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/tls/x86_64/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/tls/x86_64", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/tls/x86_64/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/tls/x86_64", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/tls/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/tls", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/x86_64/x86_64/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/x86_64/x86_64", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/x86_64/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/x86_64", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/x86_64/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph/x86_64", 0x7ffe3af2a840) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/ceph/librbd.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
stat("/usr/lib64/ceph", {st_mode=S_IFDIR|0755, st_size=42, ...}) = 0
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=61308, ...}) = 0
mmap(NULL, 61308, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fdc7c93e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/librbd.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\3\6\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=5584768, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c93c000
mmap(NULL, 7698888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7bfd0000
mprotect(0x7fdc7c504000, 2097152, PROT_NONE) = 0
mmap(0x7fdc7c704000, 126976, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x534000) = 0x7fdc7c704000
mmap(0x7fdc7c723000, 18888, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c723000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/librados.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/librados.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0PM\3\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1362768, ...}) = 0
mmap(NULL, 3459208, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7bc83000
mprotect(0x7fdc7bdc1000, 2097152, PROT_NONE) = 0
mmap(0x7fdc7bfc1000, 61440, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13e000) = 0x7fdc7bfc1000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libncurses.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libncurses.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\211\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=166968, ...}) = 0
mmap(NULL, 2262464, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7ba5a000
mprotect(0x7fdc7ba82000, 2093056, PROT_NONE) = 0
mmap(0x7fdc7bc81000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x7fdc7bc81000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libtinfo.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libtinfo.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\225\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=189288, ...}) = 0
mmap(NULL, 2285344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7b82c000
mprotect(0x7fdc7b852000, 2093056, PROT_NONE) = 0
mmap(0x7fdc7ba51000, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7fdc7ba51000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libceph-common.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\17(\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=10381024, ...}) = 0
mmap(NULL, 148956272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc72a1d000
mprotect(0x7fdc733d9000, 2097152, PROT_NONE) = 0
mmap(0x7fdc735d9000, 172032, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9bc000) = 0x7fdc735d9000
mmap(0x7fdc73603000, 136479856, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc73603000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libpthread.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200q\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=144048, ...}) = 0
mmap(NULL, 2220520, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc727fe000
mprotect(0x7fdc72817000, 2097152, PROT_NONE) = 0
mmap(0x7fdc72a17000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19000) = 0x7fdc72a17000
mmap(0x7fdc72a19000, 12776, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc72a19000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libudev.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libudev.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 B\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=145656, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c93a000
mmap(NULL, 2242832, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc725da000
mprotect(0x7fdc725fc000, 2097152, PROT_NONE) = 0
mmap(0x7fdc727fc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7fdc727fc000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/librt.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240&\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=41584, ...}) = 0
mmap(NULL, 2128384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc723d2000
mprotect(0x7fdc723d9000, 2093056, PROT_NONE) = 0
mmap(0x7fdc725d8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fdc725d8000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libblkid.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\260\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=338976, ...}) = 0
mmap(NULL, 2434936, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7217f000
mprotect(0x7fdc721cc000, 2097152, PROT_NONE) = 0
mmap(0x7fdc723cc000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4d000) = 0x7fdc723cc000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libdl.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\20\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=18392, ...}) = 0
mmap(NULL, 2109584, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc71f7b000
mprotect(0x7fdc71f7e000, 2093056, PROT_NONE) = 0
mmap(0x7fdc7217d000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fdc7217d000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\26\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=14648, ...}) = 0
mmap(NULL, 2109712, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc71d77000
mprotect(0x7fdc71d7a000, 2093056, PROT_NONE) = 0
mmap(0x7fdc71f79000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fdc71f79000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\30\n\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1932336, ...}) = 0
mmap(NULL, 4040704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7199c000
mprotect(0x7fdc71b67000, 2093056, PROT_NONE) = 0
mmap(0x7fdc71d66000, 57344, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ca000) = 0x7fdc71d66000
mmap(0x7fdc71d74000, 10240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc71d74000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\217\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1355752, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c938000
mmap(NULL, 3371560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc71664000
mprotect(0x7fdc7179b000, 2093056, PROT_NONE) = 0
mmap(0x7fdc7199a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x136000) = 0x7fdc7199a000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0000\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=101024, ...}) = 0
mmap(NULL, 2196552, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc7144b000
mprotect(0x7fdc71462000, 2097152, PROT_NONE) = 0
mmap(0x7fdc71662000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7fdc71662000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`D\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2038400, ...}) = 0
mmap(NULL, 3909496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc71090000
mprotect(0x7fdc71241000, 2097152, PROT_NONE) = 0
mmap(0x7fdc71441000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b1000) = 0x7fdc71441000
mmap(0x7fdc71447000, 14200, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc71447000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libresolv.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libresolv.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@?\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=96616, ...}) = 0
mmap(NULL, 2189472, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc70e79000
mprotect(0x7fdc70e8c000, 2097152, PROT_NONE) = 0
mmap(0x7fdc7108c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x7fdc7108c000
mmap(0x7fdc7108e000, 6304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc7108e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0000\10\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=3019840, ...}) = 0
mmap(NULL, 5133552, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc70993000
mprotect(0x7fdc70c44000, 2093056, PROT_NONE) = 0
mmap(0x7fdc70e43000, 200704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2b0000) = 0x7fdc70e43000
mmap(0x7fdc70e74000, 17648, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc70e74000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libibverbs.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/libibverbs.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20o\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=117264, ...}) = 0
mmap(NULL, 2208544, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc70777000
mprotect(0x7fdc70791000, 2097152, PROT_NONE) = 0
mmap(0x7fdc70991000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1a000) = 0x7fdc70991000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/librdmacm.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/lib64/librdmacm.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340F\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=128360, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c936000
mmap(NULL, 2224584, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc70557000
mprotect(0x7fdc70574000, 2097152, PROT_NONE) = 0
mmap(0x7fdc70774000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1d000) = 0x7fdc70774000
mmap(0x7fdc70776000, 456, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc70776000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/ceph/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@'\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=92216, ...}) = 0
mmap(NULL, 2187280, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc70340000
mprotect(0x7fdc70356000, 2093056, PROT_NONE) = 0
mmap(0x7fdc70555000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7fdc70555000
close(3)                                = 0
openat(AT_FDCWD, "/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300u\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=159424, ...}) = 0
mmap(NULL, 2263688, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc70117000
mprotect(0x7fdc7013d000, 2093056, PROT_NONE) = 0
mmap(0x7fdc7033c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7fdc7033c000
mmap(0x7fdc7033e000, 6792, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc7033e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\27\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18984, ...}) = 0
mmap(NULL, 2114136, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6ff12000
mprotect(0x7fdc6ff16000, 2093056, PROT_NONE) = 0
mmap(0x7fdc70115000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fdc70115000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\31\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=30840, ...}) = 0
mmap(NULL, 2125848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6fd0a000
mprotect(0x7fdc6fd10000, 2097152, PROT_NONE) = 0
mmap(0x7fdc6ff10000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fdc6ff10000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/libnl-route-3.so.200", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\314\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=474872, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c934000
mmap(NULL, 2578688, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6fa94000
mprotect(0x7fdc6fb00000, 2097152, PROT_NONE) = 0
mmap(0x7fdc6fd00000, 32768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6c000) = 0x7fdc6fd00000
mmap(0x7fdc6fd08000, 6400, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc6fd08000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/libnl-3.so.200", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\225\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=139240, ...}) = 0
mmap(NULL, 2234608, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6f872000
mprotect(0x7fdc6f891000, 2097152, PROT_NONE) = 0
mmap(0x7fdc6fa91000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7fdc6fa91000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\26\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=567616, ...}) = 0
mmap(NULL, 2662888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6f5e7000
mprotect(0x7fdc6f671000, 2093056, PROT_NONE) = 0
mmap(0x7fdc6f870000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x89000) = 0x7fdc6f870000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c932000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c930000
mmap(NULL, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c929000
arch_prctl(ARCH_SET_FS, 0x7fdc7c92e4c0) = 0
mprotect(0x7fdc71441000, 12288, PROT_READ) = 0
mprotect(0x7fdc6f870000, 4096, PROT_READ) = 0
mprotect(0x7fdc72a17000, 4096, PROT_READ) = 0
mprotect(0x7fdc6fa91000, 8192, PROT_READ) = 0
mprotect(0x7fdc6fd00000, 12288, PROT_READ) = 0
mprotect(0x7fdc6ff10000, 4096, PROT_READ) = 0
mprotect(0x7fdc70115000, 4096, PROT_READ) = 0
mprotect(0x7fdc7217d000, 4096, PROT_READ) = 0
mprotect(0x7fdc7033c000, 4096, PROT_READ) = 0
mprotect(0x7fdc70555000, 4096, PROT_READ) = 0
mprotect(0x7fdc70991000, 4096, PROT_READ) = 0
mprotect(0x7fdc70774000, 4096, PROT_READ) = 0
mprotect(0x7fdc70e43000, 184320, PROT_READ) = 0
mprotect(0x7fdc7108c000, 4096, PROT_READ) = 0
mprotect(0x7fdc71662000, 4096, PROT_READ) = 0
mprotect(0x7fdc7199a000, 4096, PROT_READ) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c927000
mprotect(0x7fdc71d66000, 45056, PROT_READ) = 0
mprotect(0x7fdc71f79000, 4096, PROT_READ) = 0
mprotect(0x7fdc723cc000, 20480, PROT_READ) = 0
mprotect(0x7fdc725d8000, 4096, PROT_READ) = 0
mprotect(0x7fdc727fc000, 4096, PROT_READ) = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c924000
mmap(NULL, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c91f000
mmap(NULL, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c916000
mprotect(0x7fdc735d9000, 135168, PROT_READ) = 0
mprotect(0x7fdc7ba51000, 4096, PROT_READ) = 0
mprotect(0x7fdc7bc81000, 4096, PROT_READ) = 0
mprotect(0x7fdc7bfc1000, 24576, PROT_READ) = 0
mprotect(0x7fdc7c704000, 102400, PROT_READ) = 0
mprotect(0x55ee20136000, 49152, PROT_READ) = 0
mprotect(0x7fdc7c94d000, 4096, PROT_READ) = 0
munmap(0x7fdc7c93e000, 61308)           = 0
set_tid_address(0x7fdc7c92e790)         = 28950
set_robust_list(0x7fdc7c92e7a0, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7fdc72804bf0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7fdc728112d0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7fdc72804c90, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7fdc728112d0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL)                               = 0x55ee2015c000
brk(0x55ee2017d000)                     = 0x55ee2017d000
stat("/etc/libnl/classid", {st_mode=S_IFREG|0644, st_size=1130, ...}) = 0
openat(AT_FDCWD, "/etc/libnl/classid", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1130, ...}) = 0
read(3, "################################"..., 4096) = 1130
read(3, "", 4096)                       = 0
close(3)                                = 0
statfs("/sys/fs/selinux", 0x7ffe3af2b150) = -1 ENOENT (No such file or directory)
statfs("/selinux", {f_type=OVERLAYFS_SUPER_MAGIC, f_bsize=4096, f_blocks=32941335, f_bfree=32752838, f_bavail=32752838, f_files=32941335, f_ffree=32938581, f_fsid={val=[0, 0]}, f_namelen=256, f_frsize=4096, f_flags=ST_VALID|ST_RELATIME}) = 0
openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 343
read(3, "", 1024)                       = 0
close(3)                                = 0
access("/etc/selinux/config", F_OK)     = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/proc/sys/crypto/fips_enabled", O_RDONLY) = -1 ENOENT (No such file or directory)
futex(0x7fdc70e7685c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc7217e048, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/usr/lib64/.libcrypto.so.1.1.hmac", O_RDONLY) = -1 ENOENT (No such file or directory)
futex(0x7fdc70e76dd8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc70e7673c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc70e76dcc, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc70e76dc4, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc70e76730, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc70e76dbc, FUTEX_WAKE_PRIVATE, 2147483647) = 0
access("/usr/lib64/.libcrypto.so.1.1.hmac", F_OK) = -1 ENOENT (No such file or directory)
brk(0x55ee201a7000)                     = 0x55ee201a7000
futex(0x7fdc71d746bc, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7fdc71d746c8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
brk(0x55ee201c8000)                     = 0x55ee201c8000
brk(0x55ee201e9000)                     = 0x55ee201e9000
mmap(NULL, 450560, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c8a8000
brk(0x55ee2020a000)                     = 0x55ee2020a000
brk(0x55ee2022b000)                     = 0x55ee2022b000
mmap(NULL, 897024, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc7c7cd000
munmap(0x7fdc7c8a8000, 450560)          = 0
openat(AT_FDCWD, "/usr/lib64/ceph/liblttng-ust-tracepoint.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=61308, ...}) = 0
mmap(NULL, 61308, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fdc7c93e000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/liblttng-ust-tracepoint.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\33\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=47560, ...}) = 0
mmap(NULL, 2208520, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6f3cb000
mprotect(0x7fdc6f3d6000, 2093056, PROT_NONE) = 0
mmap(0x7fdc6f5d5000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7fdc6f5d5000
mmap(0x7fdc6f5d7000, 62216, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fdc6f5d7000
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib64/liburcu-bp.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\"\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=31536, ...}) = 0
mmap(NULL, 2127312, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fdc6f1c3000
mprotect(0x7fdc6f1ca000, 2093056, PROT_NONE) = 0
mmap(0x7fdc6f3c9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7fdc6f3c9000
close(3)                                = 0
mprotect(0x7fdc6f3c9000, 4096, PROT_READ) = 0
mprotect(0x7fdc6f5d5000, 4096, PROT_READ) = 0
membarrier(MEMBARRIER_CMD_QUERY, 0)     = 0x7f (MEMBARRIER_CMD_GLOBAL|MEMBARRIER_CMD_GLOBAL_EXPEDITED|MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED|MEMBARRIER_CMD_PRIVATE_EXPEDITED|MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED|MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE|MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)
munmap(0x7fdc7c93e000, 61308)           = 0
brk(0x55ee2024c000)                     = 0x55ee2024c000
futex(0x7fdc716631e0, FUTEX_WAKE_PRIVATE, 2147483647) = 0
uname({sysname="Linux", nodename="nid001388", ...}) = 0
mmap(NULL, 10883072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fdc6e762000
brk(0x55ee20271000)                     = 0x55ee20271000
getrandom("\xe6", 1, 0)                 = 1
prctl(PR_GET_NAME, "rbd")               = 0
stat("/etc/ceph/ceph.conf", {st_mode=S_IFREG|0644, st_size=138, ...}) = 0
openat(AT_FDCWD, "/etc/ceph/ceph.conf", O_RDONLY) = 3
read(3, "# Ansible managed\n[global]\n  fsi"..., 8191) = 138
read(3, "", 8191)                       = 0
close(3)                                = 0
rt_sigprocmask(SIG_BLOCK, [PIPE], [], 8) = 0
mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7fdc6df61000
mprotect(0x7fdc6df62000, 8388608, PROT_READ|PROT_WRITE) = 0
clone(child_stack=0x7fdc6e75c230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[28952], tls=0x7fdc6e761700, child_tidptr=0x7fdc6e7619d0) = 28952
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
strace: Process 28952 attached
[pid 28950] rt_sigprocmask(SIG_BLOCK, [PIPE], NULL, 8) = 0
[pid 28952] set_robust_list(0x7fdc6e7619e0, 24 <unfinished ...>
[pid 28950] rt_sigaction(SIGSEGV, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0},  <unfinished ...>
[pid 28952] <... set_robust_list resumed>) = 0
[pid 28950] <... rt_sigaction resumed>{sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28950] rt_sigaction(SIGABRT, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0},  <unfinished ...>
[pid 28952] gettid( <unfinished ...>
[pid 28950] <... rt_sigaction resumed>{sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28952] <... gettid resumed>)       = 28952
[pid 28950] rt_sigaction(SIGBUS, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0},  <unfinished ...>
[pid 28952] prctl(PR_SET_NAME, "log" <unfinished ...>
[pid 28950] <... rt_sigaction resumed>{sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28952] <... prctl resumed>)        = 0
[pid 28950] rt_sigaction(SIGILL, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28950] rt_sigaction(SIGFPE, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0},  <unfinished ...>
[pid 28952] futex(0x55ee2023f338, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
[pid 28950] <... rt_sigaction resumed>{sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28950] rt_sigaction(SIGXCPU, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28950] rt_sigaction(SIGXFSZ, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28950] rt_sigaction(SIGSYS, {sa_handler=0x55ee1fe930b0, sa_mask=[], sa_flags=SA_RESTORER|SA_NODEFER|SA_RESETHAND, sa_restorer=0x7fdc728112d0}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
[pid 28950] getuid()                    = 0
[pid 28950] prctl(PR_SET_DUMPABLE, SUID_DUMP_USER) = 0
[pid 28950] prctl(PR_SET_THP_DISABLE, 1, 0, 0, 0) = 0
[pid 28950] uname({sysname="Linux", nodename="nid001388", ...}) = 0
[pid 28950] rt_sigprocmask(SIG_BLOCK, [PIPE], [PIPE], 8) = 0
[pid 28950] mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7fdc6d760000
[pid 28950] mprotect(0x7fdc6d761000, 8388608, PROT_READ|PROT_WRITE) = 0
[pid 28950] clone(child_stack=0x7fdc6df5b230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTIDstrace: Process 28953 attached
, parent_tid=[28953], tls=0x7fdc6df60700, child_tidptr=0x7fdc6df609d0) = 28953
[pid 28953] set_robust_list(0x7fdc6df609e0, 24 <unfinished ...>
[pid 28950] rt_sigprocmask(SIG_SETMASK, [PIPE],  <unfinished ...>
[pid 28953] <... set_robust_list resumed>) = 0
[pid 28950] <... rt_sigprocmask resumed>NULL, 8) = 0
[pid 28953] gettid( <unfinished ...>
[pid 28950] getpid( <unfinished ...>
[pid 28953] <... gettid resumed>)       = 28953
[pid 28950] <... getpid resumed>)       = 28950
[pid 28950] gettid( <unfinished ...>
[pid 28953] prctl(PR_SET_NAME, "service" <unfinished ...>
[pid 28950] <... gettid resumed>)       = 28950
[pid 28953] <... prctl resumed>)        = 0
[pid 28950] getrandom("\x2d\xc0\x08\xe1\x93\x02\xf1\xeb\x01\xce\x22\x18\xbf\x4d\xd6\x34", 16, GRND_NONBLOCK) = 16
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404516, tv_nsec=731275291}, FUTEX_BITSET_MATCH_ANY <unfinished ...>
[pid 28950] getrandom("\x6e\x86\xe2\xdf\x50\x92\x4a\x20\x98\x11\x7c\xc0\xc7\x8b\x75\x5b", 16, GRND_NONBLOCK) = 16
[pid 28950] access("/sys/subsystem", F_OK) = -1 ENOENT (No such file or directory)
[pid 28950] open("/sys/bus", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
[pid 28950] fstat(3, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
[pid 28950] getdents(3, /* 25 entries */, 32768) = 696
[pid 28950] open("/sys/bus/rbd/devices/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4
[pid 28950] fstat(4, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
[pid 28950] brk(0x55ee20296000)         = 0x55ee20296000
[pid 28950] getdents(4, /* 2 entries */, 32768) = 48
[pid 28950] getdents(4, /* 0 entries */, 32768) = 0
[pid 28950] close(4)                    = 0
[pid 28950] getdents(3, /* 0 entries */, 32768) = 0
[pid 28950] close(3)                    = 0
[pid 28950] open("/sys/class", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
[pid 28950] fstat(3, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
[pid 28950] getdents(3, /* 48 entries */, 32768) = 1472
[pid 28950] getdents(3, /* 0 entries */, 32768) = 0
[pid 28950] close(3)                    = 0
[pid 28950] access("/sys/subsystem", F_OK) = -1 ENOENT (No such file or directory)
[pid 28950] open("/sys/bus", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
[pid 28950] fstat(3, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
[pid 28950] getdents(3, /* 25 entries */, 32768) = 696
[pid 28950] open("/sys/bus/rbd/devices/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4
[pid 28950] fstat(4, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
[pid 28950] getdents(4, /* 2 entries */, 32768) = 48
[pid 28950] getdents(4, /* 0 entries */, 32768) = 0
[pid 28950] close(4)                    = 0
[pid 28950] getdents(3, /* 0 entries */, 32768) = 0
[pid 28950] close(3)                    = 0
[pid 28950] open("/sys/class", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 3
[pid 28950] fstat(3, {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
[pid 28950] getdents(3, /* 48 entries */, 32768) = 1472
[pid 28950] getdents(3, /* 0 entries */, 32768) = 0
[pid 28950] close(3)                    = 0
[pid 28950] access("/sys/bus/rbd", F_OK) = 0
[pid 28950] openat(AT_FDCWD, "/etc/ceph/ceph.client.noir.keyring", O_RDONLY|O_CLOEXEC) = 3
[pid 28950] close(3)                    = 0
[pid 28950] openat(AT_FDCWD, "/etc/ceph/ceph.client.noir.keyring", O_RDONLY|O_CLOEXEC) = 3
[pid 28950] fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
[pid 28950] read(3, "# Ansible managed\n[client.noir]\n"..., 81) = 81
[pid 28950] close(3)                    = 0
[pid 28950] futex(0x55ee2023f338, FUTEX_WAKE_PRIVATE, 2147483647) = 1
[pid 28952] <... futex resumed>)        = 0
[pid 28952] futex(0x55ee2023f290, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
[pid 28950] add_key("ceph", "client.noir", "--REDACTED--", 28, KEY_SPEC_PROCESS_KEYRING <unfinished ...>
[pid 28952] <... futex resumed>)        = 0
[pid 28952] futex(0x55ee2023f33c, FUTEX_WAIT_PRIVATE, 0, NULL <unfinished ...>
[pid 28950] <... add_key resumed>)      = 1053079752
[pid 28950] socket(AF_NETLINK, SOCK_RAW|SOCK_CLOEXEC|SOCK_NONBLOCK, NETLINK_KOBJECT_UEVENT) = 3
[pid 28950] setsockopt(3, SOL_SOCKET, SO_RCVBUFFORCE, [1048576], 4) = 0
[pid 28950] setsockopt(3, SOL_SOCKET, SO_ATTACH_FILTER, {len=13, filter=0x7ffe3af294f0}, 16) = 0
[pid 28950] bind(3, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=0x000001}, 12) = 0
[pid 28950] getsockname(3, {sa_family=AF_NETLINK, nl_pid=28950, nl_groups=0x000001}, [12]) = 0
[pid 28950] setsockopt(3, SOL_SOCKET, SO_PASSCRED, [1], 4) = 0
[pid 28950] pipe2([4, 5], O_NONBLOCK)   = 0
[pid 28950] mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7fdc6cf5f000
[pid 28950] mprotect(0x7fdc6cf60000, 8388608, PROT_READ|PROT_WRITE) = 0
[pid 28950] clone(child_stack=0x7fdc6d75a230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[28954], tls=0x7fdc6d75f700, child_tidptr=0x7fdc6d75f9d0) = 28954
[pid 28950] poll([{fd=4, events=POLLIN}, {fd=3, events=POLLIN}], 2, -1strace: Process 28954 attached
 <unfinished ...>
[pid 28954] set_robust_list(0x7fdc6d75f9e0, 24) = 0
[pid 28954] prctl(PR_SET_NAME, "mapper") = 0
[pid 28954] mmap(NULL, 134217728, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = 0x7fdc64f5f000
[pid 28954] munmap(0x7fdc64f5f000, 50991104) = 0
[pid 28954] munmap(0x7fdc6c000000, 16117760) = 0
[pid 28954] mprotect(0x7fdc68000000, 135168, PROT_READ|PROT_WRITE) = 0
[pid 28954] openat(AT_FDCWD, "/sys/bus/rbd/add_single_major", O_WRONLY) = 6
[pid 28954] write(6, "148.187.20.141:6789 name=noir,ke"..., 72 <unfinished ...>
[pid 28953] <... futex resumed>)        = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404521, tv_nsec=731561766}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404526, tv_nsec=731874611}, FUTEX_BITSET_MATCH_ANY
) = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404531, tv_nsec=732152640}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404536, tv_nsec=732430988}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404541, tv_nsec=732728744}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404546, tv_nsec=733000770}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)
[pid 28953] futex(0x55ee20254b70, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 28953] futex(0x55ee20254bc0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, {tv_sec=1653404551, tv_nsec=733303536}, FUTEX_BITSET_MATCH_ANY) = -1 ETIMEDOUT (Connection timed out)




________________________________
From: Ilya Dryomov <idryomov@xxxxxxxxx>
Sent: Tuesday, May 24, 2022 4:25:33 PM
To: Sopena Ballesteros Manuel
Cc: ceph-users@xxxxxxx
Subject: Re:  rbd command hangs

On Tue, May 24, 2022 at 3:57 PM Sopena Ballesteros Manuel
<manuel.sopena@xxxxxxx> wrote:
>
> Dear ceph user community,
>
>
> I am trying to install and configure a node with a ceph cluster. The linux kernel we have does not include the rbd kernel module, hence we installed if ourselves:
>
>
> zypper install -y ceph-common > 15
> zypper install -y kernel-source = 5.3.18-24.75_10.0.189_2.1_20.4__g0388af5bc3.shasta
> cp /boot/config-5.3.18-24.75_10.0.189-cray_shasta_c /usr/src/linux/.config
> chown root:root /usr/src/linux/.config
> chown 0644 /usr/src/linux/.config
> cd /usr/src/linux
> sed -i 's/^# CONFIG_BLK_DEV_RBD is not set/CONFIG_BLK_DEV_RBD=m/g' .config && echo 'CONFIG_TCM_RBD=m' >> .config
> make drivers/block/rbd.ko
> cp /usr/src/linux/drivers/block/rbd.ko /lib/modules/5.3.18-24.75_10.0.189-cray_shasta_c/extra/rbd.ko
> chown root:root /lib/modules/5.3.18-24.75_10.0.189-cray_shasta_c/extra/rbd.ko
> chown 0644 /lib/modules/5.3.18-24.75_10.0.189-cray_shasta_c/extra/rbd.ko
>
>
> My issue is that rbd command sometimes hangs and we don't know why, this does not occur all the time but quite frequently. I google bit but could not find any relevant solution so I am looking for advice.
>
>
> What could cause rbd command to hang?

Hi Manuel,

Did you check if the RBD device gets mapped anyway?  If the mapping
succeeds despite the hang, it is probably hanging waiting for udev to
do its job.  It could be somehow related to the stripped down kernel
you are using or, if you are running "rbd map" from a container, there
may be issues with netlink event propagation.  Try "noudev" mapping
option:

$ rbd map -o noudev noir-nvme-meta/nid001388

>
>
> Below is an strace of when we try to run an rbd command:
>
>
> nid001388:~ # strace rbd -n client.noir map noir-nvme-meta/nid001388
> execve("/usr/bin/rbd", ["rbd", "-n", "client.noir", "map", "noir-nvme-meta/nid001388"], 0x7ffe8c35b7b0 /* 62 vars */) = 0
>
> [...]
>
> add_key("ceph", "client.noir", "--REDACTED--", 28, KEY_SPEC_PROCESS_KEYRING) = 201147173
> access("/run/udev/control", F_OK)       = 0
> socket(AF_NETLINK, SOCK_RAW|SOCK_CLOEXEC|SOCK_NONBLOCK, NETLINK_KOBJECT_UEVENT) = 3
> setsockopt(3, SOL_SOCKET, SO_RCVBUFFORCE, [1048576], 4) = 0
> setsockopt(3, SOL_SOCKET, SO_ATTACH_FILTER, {len=13, filter=0x7ffd2f2179c0}, 16) = 0
> bind(3, {sa_family=AF_NETLINK, nl_pid=0, nl_groups=0x000002}, 12) = 0
> getsockname(3, {sa_family=AF_NETLINK, nl_pid=21421, nl_groups=0x000002}, [12]) = 0
> setsockopt(3, SOL_SOCKET, SO_PASSCRED, [1], 4) = 0
> pipe2([4, 5], O_NONBLOCK)               = 0
> mmap(NULL, 8392704, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7ff1e954f000
> mprotect(0x7ff1e9550000, 8388608, PROT_READ|PROT_WRITE) = 0
> clone(child_stack=0x7ff1e9d4a230, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tid=[21425], tls=0x7ff1e9d4f700, child_tidptr=0x7ff1e9d4f9d0) = 21425
> poll([{fd=4, events=POLLIN}, {fd=3, events=POLLIN}], 2, -1

This doesn't tell anything definitive as the actual mapping is done
from a thread.  Pass -f to strace to also trace child processes.

Thanks,

                Ilya
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx



[Index of Archives]     [Information on CEPH]     [Linux Filesystem Development]     [Ceph Development]     [Ceph Large]     [Ceph Dev]     [Linux USB Development]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]     [xfs]


  Powered by Linux