Hello,
My problem description:
On cephfs client 1:
Copy a directory "base" which contains hundreds of rpm files to the
mounted cephfs directory.
then , change to cephfs directory and execute "ls -l base", it returns
quickly. This is no problem.
On the other cephfs client,
Change to the mounted cephfs directory and execute "ls -l base", it
blocks for long time.
Execute "strace ls -l base", we can find it blocks in lstat files (about
5 seconds blocking for each file).
Is there anybody met this problem?
==========================================================
cephfs client information:
OS: CentOS Linux release 7.2.1511 (Core)
kernel: Linux 3.10.0-327.el7.x86_64
mount type: with ceph kernel drvier
==========================================================
ceph cluster information:
OS: CentOS Linux release 7.2.1511 (Core)
kernel: Linux 3.10.0-327.el7.x86_64
ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous
(stable)
ceph cluster node1: 1 mon, 1 mgr, 1 mds, 4 osd
ceph cluster node2: 1 mon, 1 mgr, 1 mds, 4 osd
ceph cluster node3: 1 mon, 1 mgr, 1 mds, 4 osd
Each osd is deployed with bluestore type (4TB HDD block device + 240GB
SSD block.db device).
public network: port 1 of 56Gb/s Dual Port Infiniband, IPoIB
cluster network: port 2 of 56Gb/s Dual Port Infiniband, IPoIB
ceph.conf:
-------------------------
[global]
fsid = 86887b08-6ff4-4ec1-a622-64bc688d1f2f
mon_initial_members = storage-0, storage-1, storage-2
mon_host = 10.0.30.11,10.0.30.12,10.0.30.13
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public_network = 10.0.30.0/24
cluster_network = 10.0.40.0/24
osd_pool_default_size = 2
osd_pool_default_min_size = 1
osd_objectstore = bluestore
bluestore_block_db_size = 236223201280
==========================================================
strace log:
-------------------------
[root@computing-1 cephfs]# strace ls -l base
execve("/usr/bin/ls", ["ls", "-l", "base"], [/* 24 vars */]) = 0
brk(0) = 0x155f000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b91000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=53575, ...}) = 0
mmap(NULL, 53575, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c65b83000
close(3) = 0
open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300j\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=155744, ...}) = 0
mmap(NULL, 2255216, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c6574a000
mprotect(0x7f8c6576e000, 2093056, PROT_NONE) = 0
mmap(0x7f8c6596d000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f8c6596d000
mmap(0x7f8c6596f000, 6512, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c6596f000
close(3) = 0
open("/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0
\26\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=20024, ...}) = 0
mmap(NULL, 2114112, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c65545000
mprotect(0x7f8c65549000, 2093056, PROT_NONE) = 0
mmap(0x7f8c65748000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f8c65748000
close(3) = 0
open("/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\37\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=37056, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b82000
mmap(NULL, 2130560, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c6533c000
mprotect(0x7f8c65343000, 2097152, PROT_NONE) = 0
mmap(0x7f8c65543000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f8c65543000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0
\34\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2107816, ...}) = 0
mmap(NULL, 3932736, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c64f7b000
mprotect(0x7f8c65131000, 2097152, PROT_NONE) = 0
mmap(0x7f8c65331000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b6000) = 0x7f8c65331000
mmap(0x7f8c65337000, 16960, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c65337000
close(3) = 0
open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\25\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=398272, ...}) = 0
mmap(NULL, 2490888, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c64d1a000
mprotect(0x7f8c64d7a000, 2093056, PROT_NONE) = 0
mmap(0x7f8c64f79000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5f000) = 0x7f8c64f79000
close(3) = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19512, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b81000
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c64b16000
mprotect(0x7f8c64b19000, 2093056, PROT_NONE) = 0
mmap(0x7f8c64d18000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f8c64d18000
close(3) = 0
open("/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=19888, ...}) = 0
mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c64911000
mprotect(0x7f8c64915000, 2093056, PROT_NONE) = 0
mmap(0x7f8c64b14000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f8c64b14000
close(3) = 0
open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240l\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=142296, ...}) = 0
mmap(NULL, 2208864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c646f5000
mprotect(0x7f8c6470b000, 2097152, PROT_NONE) = 0
mmap(0x7f8c6490b000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f8c6490b000
mmap(0x7f8c6490d000, 13408, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c6490d000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b80000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b7e000
arch_prctl(ARCH_SET_FS, 0x7f8c65b7e800) = 0
mprotect(0x7f8c65331000, 16384, PROT_READ) = 0
mprotect(0x7f8c6490b000, 4096, PROT_READ) = 0
mprotect(0x7f8c64b14000, 4096, PROT_READ) = 0
mprotect(0x7f8c64d18000, 4096, PROT_READ) = 0
mprotect(0x7f8c64f79000, 4096, PROT_READ) = 0
mprotect(0x7f8c65543000, 4096, PROT_READ) = 0
mprotect(0x7f8c65748000, 4096, PROT_READ) = 0
mprotect(0x7f8c6596d000, 4096, PROT_READ) = 0
mprotect(0x61a000, 4096, PROT_READ) = 0
mprotect(0x7f8c65b92000, 4096, PROT_READ) = 0
munmap(0x7f8c65b83000, 53575) = 0
set_tid_address(0x7f8c65b7ead0) = 121484
set_robust_list(0x7f8c65b7eae0, 24) = 0
rt_sigaction(SIGRTMIN, {0x7f8c646fb780, [], SA_RESTORER|SA_SIGINFO,
0x7f8c64704100}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f8c646fb810, [],
SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f8c64704100}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
statfs("/sys/fs/selinux", 0x7ffeb752e770) = -1 ENOENT (No such file or
directory)
statfs("/selinux", 0x7ffeb752e770) = -1 ENOENT (No such file or
directory)
brk(0) = 0x155f000
brk(0x1580000) = 0x1580000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b90000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 1024) = 377
stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned",
0x7ffeb752e650) = -1 ENOENT (No such file or directory)
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7f8c65b90000, 4096) = 0
access("/etc/selinux/config", F_OK) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=106065056, ...}) = 0
mmap(NULL, 106065056, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c5e1ce000
close(3) = 0
ioctl(1, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS,
{B38400 opost isig icanon echo ...}) = 0
ioctl(1, TIOCGWINSZ, {ws_row=68, ws_col=252, ws_xpixel=0, ws_ypixel=0}) = 0
open("/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2502, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b90000
read(3, "# Locale name alias data base.\n#"..., 4096) = 2502
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7f8c65b90000, 4096) = 0
open("/usr/share/locale/en_US.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) =
-1 ENOENT (No such file or directory)
open("/usr/share/locale/en_US.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1
ENOENT (No such file or directory)
open("/usr/share/locale/en_US/LC_TIME/coreutils.mo", O_RDONLY) = -1
ENOENT (No such file or directory)
open("/usr/share/locale/en.UTF-8/LC_TIME/coreutils.mo", O_RDONLY) = -1
ENOENT (No such file or directory)
open("/usr/share/locale/en.utf8/LC_TIME/coreutils.mo", O_RDONLY) = -1
ENOENT (No such file or directory)
open("/usr/share/locale/en/LC_TIME/coreutils.mo", O_RDONLY) = -1 ENOENT
(No such file or directory)
stat("/etc/sysconfig/64bit_strstr_via_64bit_strstr_sse2_unaligned",
0x7ffeb752e2b0) = -1 ENOENT (No such file or directory)
open("/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=26254, ...}) = 0
mmap(NULL, 26254, PROT_READ, MAP_SHARED, 3, 0) = 0x7f8c65b8a000
close(3) = 0
futex(0x7f8c65336a80, FUTEX_WAKE_PRIVATE, 2147483647) = 0
lstat("base", {st_mode=S_IFDIR|0777, st_size=3929293109, ...}) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
-1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
-1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1717, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b89000
read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1717
read(3, "", 4096) = 0
close(3) = 0
munmap(0x7f8c65b89000, 4096) = 0
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=53575, ...}) = 0
mmap(NULL, 53575, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f8c65b70000
close(3) = 0
open("/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\"\0\0\0\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0755, st_size=57824, ...}) = 0
mmap(NULL, 2168952, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f8c5dfbc000
mprotect(0x7f8c5dfc7000, 2093056, PROT_NONE) = 0
mmap(0x7f8c5e1c6000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa000) = 0x7f8c5e1c6000
mmap(0x7f8c5e1c8000, 22648, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f8c5e1c8000
close(3) = 0
mprotect(0x7f8c5e1c6000, 4096, PROT_READ) = 0
munmap(0x7f8c65b70000, 53575) = 0
open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=1330, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b89000
read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 1330
close(3) = 0
munmap(0x7f8c65b89000, 4096) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
-1 ENOENT (No such file or directory)
close(3) = 0
socket(PF_LOCAL, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_LOCAL, sun_path="/var/run/nscd/socket"}, 110) =
-1 ENOENT (No such file or directory)
close(3) = 0
open("/etc/group", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=597, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
= 0x7f8c65b89000
read(3, "root:x:0:\nbin:x:1:\ndaemon:x:2:\ns"..., 4096) = 597
close(3) = 0
munmap(0x7f8c65b89000, 4096) = 0
openat(AT_FDCWD, "base", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 533 entries */, 32768) = 32720
lstat("base/dbus-glib-0.100-7.el7.x86_64.rpm", {st_mode=S_IFREG|0777,
st_size=104756, ...}) = 0
lstat("base/tomcat-admin-webapps-7.0.54-2.el7_1.noarch.rpm",
{st_mode=S_IFREG|0777, st_size=38872, ...}) = 0
lstat("base/hunspell-gl-0.20080515-8.el7.noarch.rpm",
{st_mode=S_IFREG|0777, st_size=212276, ...}) = 0
lstat("base/libreoffice-langpack-de-4.3.7.2-5.el7.x86_64.rpm",
{st_mode=S_IFREG|0777, st_size=7570712, ...}) = 0
lstat("base/paktype-naskh-basic-fonts-4.1-3.el7.noarch.rpm",
{st_mode=S_IFREG|0777, st_size=436532, ...}) = 0
lstat("base/openlmi-networking-0.3.0-3.el7.x86_64.rpm",
{st_mode=S_IFREG|0777, st_size=148344, ...}) = 0
lstat("base/kdenetwork-kopete-libs-4.10.5-8.el7_0.x86_64.rpm",
{st_mode=S_IFREG|0777, st_size=1191908, ...}) = 0
lstat("base/libwmf-0.2.8.4-41.el7_1.x86_64.rpm", {st_mode=S_IFREG|0777,
st_size=137716, ...}) = 0
lstat("base/texlive-texconfig-bin-svn27344.0-38.20130427_r30134.el7.noarch.rpm",
{st_mode=S_IFREG|0777, st_size=16908, ...}) = 0
lstat("base/gcc-c++-4.8.5-4.el7.x86_64.rpm", {st_mode=S_IFREG|0777,
st_size=7508064, ...}) = 0
==========================================================
--
Best Regards
Jian Zhang
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
