Darryl,
We did notice an issue today in which setting xattrs is disallowed on
the root CephFS directory. I (still :) don't see any evidence in the
given strace that that was the cause of the problem, but if you wanted
to test it out with the newest master branch (or just cherry-pick the
relevant commits 9f82ae60fac30391dfa9d17d2fc014bf9e21f387 and
79f09bf33e3f4b6815d854fa0ce30b006b1e3e74 on to whatever you're using,
if you'd like a stable release) you could test it.
-Greg
On Wed, Feb 20, 2013 at 10:00 AM, Gregory Farnum <greg@xxxxxxxxxxx> wrote:
> [ Re-added the list for archival and informational purposes. ]
>
> I don't see any reference to xattr functions in this strace, and
> nothing's returning EOPNOTSUPP — although there are several ENOENTs on
> paths like "/var/run/nscd/socket". I think it's misconfigured somehow,
> but I'm afraid I don't know enough about SELinux to help you figure
> out what — I had to google just for the references I made yesterday.
> ;)
> -Greg
>
> On Tue, Feb 19, 2013 at 10:11 PM, Darryl Bond <dbond@xxxxxxxxxxxxx> wrote:
>> With setenforce 0
>> # ls -lZ afile
>> -rw-r--r-- root root ? afile
>>
>> # strace ls -lZ afile
>> execve("/bin/ls", ["ls", "-lZ", "afile"], [/* 25 vars */]) = 0
>> brk(0) = 0x228c000
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851cd0e000
>> access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or
>> directory)
>> open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=107205, ...}) = 0
>> mmap(NULL, 107205, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f851ccf3000
>> close(3) = 0
>> open("/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3
>> read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0pa o;\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=136440, ...}) = 0
>> mmap(0x3b6f200000, 2234408, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3b6f200000
>> mprotect(0x3b6f21f000, 2093056, PROT_NONE) = 0
>> mmap(0x3b6f41e000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x3b6f41e000
>> mmap(0x3b6f420000, 6184, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3b6f420000
>> close(3) = 0
>> open("/lib64/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\"\200\0267\0\0\0"...,
>> 832)
>> = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=47624, ...}) = 0
>> mmap(0x3716800000, 2128984, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3716800000
>> mprotect(0x3716807000, 2093056, PROT_NONE) = 0
>> mmap(0x3716a06000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x3716a06000
>> close(3) = 0
>> open("/lib64/libcap.so.2", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\25@\0327\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=21392, ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851ccf2000
>> mmap(0x371a400000, 2114080, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x371a400000
>> mprotect(0x371a404000, 2093056, PROT_NONE) = 0
>> mmap(0x371a603000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x371a603000
>> close(3) = 0
>> open("/lib64/libacl.so.1", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\37\30017\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=39192, ...}) = 0
>> mmap(0x3731c00000, 2130560, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3731c00000
>> mprotect(0x3731c07000, 2097152, PROT_NONE) = 0
>> mmap(0x3731e07000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x3731e07000
>> close(3) = 0
>> open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\33\302\0257\0\0\0"...,
>> 832)
>> = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=2071376, ...}) = 0
>> mmap(0x3715c00000, 3896312, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3715c00000
>> mprotect(0x3715dad000, 2097152, PROT_NONE) = 0
>> mmap(0x3715fad000, 24576, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ad000) = 0x3715fad000
>> mmap(0x3715fb3000, 17400, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3715fb3000
>> close(3) = 0
>> open("/lib64/libpcre.so.1", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\35`o;\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=388152, ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851ccf1000
>> mmap(0x3b6f600000, 2478664, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3b6f600000
>> mprotect(0x3b6f65c000, 2097152, PROT_NONE) = 0
>> mmap(0x3b6f85c000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5c000) = 0x3b6f85c000
>> close(3) = 0
>> open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0267\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=22440, ...}) = 0
>> mmap(0x3716000000, 2109736, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3716000000
>> mprotect(0x3716003000, 2093056, PROT_NONE) = 0
>> mmap(0x3716202000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x3716202000
>> close(3) = 0
>> open("/lib64/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360j@\0267\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=144552, ...}) = 0
>> mmap(0x3716400000, 2208808, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3716400000
>> mprotect(0x3716416000, 2097152, PROT_NONE) = 0
>> mmap(0x3716616000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x3716616000
>> mmap(0x3716618000, 13352, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3716618000
>> close(3) = 0
>> open("/lib64/libattr.so.1", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\23\300,7\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=22136, ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851ccf0000
>> mmap(0x372cc00000, 2113880, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x372cc00000
>> mprotect(0x372cc04000, 2093056, PROT_NONE) = 0
>> mmap(0x372ce03000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x372ce03000
>> close(3) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851ccef000
>> mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851cced000
>> arch_prctl(ARCH_SET_FS, 0x7f851cced7c0) = 0
>> mprotect(0x61a000, 4096, PROT_READ) = 0
>> mprotect(0x3b6f41e000, 4096, PROT_READ) = 0
>> mprotect(0x3716a06000, 4096, PROT_READ) = 0
>> mprotect(0x371a603000, 4096, PROT_READ) = 0
>> mprotect(0x3731e07000, 4096, PROT_READ) = 0
>> mprotect(0x3715fad000, 16384, PROT_READ) = 0
>> mprotect(0x3b6f85c000, 4096, PROT_READ) = 0
>> mprotect(0x3716202000, 4096, PROT_READ) = 0
>> mprotect(0x3715a20000, 4096, PROT_READ) = 0
>> mprotect(0x3716616000, 4096, PROT_READ) = 0
>> mprotect(0x372ce03000, 4096, PROT_READ) = 0
>> munmap(0x7f851ccf3000, 107205) = 0
>> set_tid_address(0x7f851cceda90) = 18454
>> set_robust_list(0x7f851ccedaa0, 24) = 0
>> rt_sigaction(SIGRTMIN, {0x3716406650, [], SA_RESTORER|SA_SIGINFO,
>> 0x371640f000}, NULL, 8) = 0
>> rt_sigaction(SIGRT_1, {0x37164066d0, [],
>> SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x371640f000}, NULL, 8) = 0
>> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
>> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
>> statfs("/sys/fs/selinux", {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0,
>> f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0},
>> f_namelen=255, f_frsize=4096}) = 0
>> statfs("/sys/fs/selinux", {f_type=0xf97cff8c, f_bsize=4096, f_blocks=0,
>> f_bfree=0, f_bavail=0, f_files=0, f_ffree=0, f_fsid={0, 0},
>> f_namelen=255, f_frsize=4096}) = 0
>> stat("/sys/fs/selinux", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
>> brk(0) = 0x228c000
>> brk(0x22ad000) = 0x22ad000
>> open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=104789808, ...}) = 0
>> mmap(NULL, 104789808, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f85168fd000
>> close(3) = 0
>> ioctl(1, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS,
>> {B38400 opost isig icanon echo ...}) = 0
>> ioctl(1, TIOCGWINSZ, {ws_row=64, ws_col=227, ws_xpixel=0, ws_ypixel=0}) = 0
>> lstat("afile", {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
>> socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
>> connect(3, {sa_family=AF_FILE, sun_path="/var/run/nscd/socket"}, 110) =
>> -1 ENOENT (No such file or directory)
>> close(3) = 0
>> socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
>> connect(3, {sa_family=AF_FILE, sun_path="/var/run/nscd/socket"}, 110) =
>> -1 ENOENT (No such file or directory)
>> close(3) = 0
>> open("/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=1717, ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851cd0d000
>> read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1717
>> read(3, "", 4096) = 0
>> close(3) = 0
>> munmap(0x7f851cd0d000, 4096) = 0
>> open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=107205, ...}) = 0
>> mmap(NULL, 107205, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f851ccf3000
>> close(3) = 0
>> open("/lib64/libnss_files.so.2", O_RDONLY|O_CLOEXEC) = 3
>> read(3,
>> "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340!\0\0\0\0\0\0"...,
>> 832) = 832
>> fstat(3, {st_mode=S_IFREG|0755, st_size=62416, ...}) = 0
>> mmap(NULL, 2148456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>> 0) = 0x7f85166f0000
>> mprotect(0x7f85166fc000, 2093056, PROT_NONE) = 0
>> mmap(0x7f85168fb000, 8192, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x7f85168fb000
>> close(3) = 0
>> mprotect(0x7f85168fb000, 4096, PROT_READ) = 0
>> munmap(0x7f851ccf3000, 107205) = 0
>> open("/etc/passwd", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=2091, ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851cd0d000
>> read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 2091
>> close(3) = 0
>> munmap(0x7f851cd0d000, 4096) = 0
>> socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
>> connect(3, {sa_family=AF_FILE, sun_path="/var/run/nscd/socket"}, 110) =
>> -1 ENOENT (No such file or directory)
>> close(3) = 0
>> socket(PF_FILE, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
>> connect(3, {sa_family=AF_FILE, sun_path="/var/run/nscd/socket"}, 110) =
>> -1 ENOENT (No such file or directory)
>> close(3) = 0
>> open("/etc/group", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=796, ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851cd0d000
>> read(3, "root:x:0:\nbin:x:1:\ndaemon:x:2:\ns"..., 4096) = 796
>> close(3) = 0
>> munmap(0x7f851cd0d000, 4096) = 0
>> fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
>> mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f851cd0d000
>> write(1, "-rw-r--r-- root root ? "..., 60-rw-r--r-- root root
>> ? afile
>> ) = 60
>> close(1) = 0
>> munmap(0x7f851cd0d000, 4096) = 0
>> close(2) = 0
>> exit_group(0) = ?
>> +++ exited with 0 +++
>>
>>
>> On 02/20/13 11:36, Gregory Farnum wrote:
>>>
>>> Hmm, SELinux appears to use the "security.selinux" xattr namespace,
>>> and "security.*" is allowed through Ceph's filters. Can you check and
>>> make sure that it's in fact using the xattr labeling scheme and not
>>> something else? Maybe strace the process and check exactly which
>>> syscall fails in what way.
>>> -Greg
>>>
>>> On Mon, Feb 18, 2013 at 4:19 PM, Darryl Bond <dbond@xxxxxxxxxxxxx> wrote:
>>>>
>>>> I believe that it was the kernel client. I had installed the rpms from
>>>> the Ceph download (0.56.3)
>>>> mount -t cephfs ...
>>>>
>>>> I was using 3.7.7 yesterday.
>>>>
>>>> Darryl
>>>>
>>>>
>>>>
>>>> On 02/19/13 10:12, Gregory Farnum wrote:
>>>>>
>>>>> This is using the kernel client? What kernel version does Fedora 18 use?
>>>>>
>>>>> I would expect this to work fine as CephFS enables xattrs by default,
>>>>> but
>>>>> perhaps we've made a mistake in filtering somewhere…
>>>>> -Greg
>>>>>
>>>>>
>>>>> On Sunday, February 17, 2013 at 3:56 PM, Darryl Bond wrote:
>>>>>
>>>>>> Hello,
>>>>>> I have mounted a cephfs filesystem on Fedora18 client. I am using
>>>>>> SELinux and get permission denied unless I setenforce 0.
>>>>>> The filesystem cannot be labelled to allow it to work with SELinux.
>>>>>> # chcon --reference=/var /mnt
>>>>>> chcon: failed to change context of /mnt to system_u:object_r:var_t:s0:
>>>>>> Operation not supported
>>>>>>
>>>>>>
>>>>>> I can't see any options to enable extended attributes in MDS or
>>>>>> mount.ceph
>>>>>>
>>>>>> Regards
>>>>>> Darryl
>>>>>
>>>>>
>>>>
>>>> The contents of this electronic message and any attachments are intended
>>>> only for the addressee and may contain legally privileged, personal,
>>>> sensitive or confidential information. If you are not the intended
>>>> addressee, and have received this email, any transmission, distribution,
>>>> downloading, printing or photocopying of the contents of this message or
>>>> attachments is strictly prohibited. Any legal privilege or
>>>> confidentiality
>>>> attached to this message and attachments is not waived, lost or destroyed
>>>> by
>>>> reason of delivery to any person other than intended addressee. If you
>>>> have
>>>> received this message and are not the intended addressee you should
>>>> notify
>>>> the sender by return email and destroy all copies of the message and any
>>>> attachments. Unless expressly attributed, the views expressed in this
>>>> email
>>>> do not necessarily represent the views of the company.
>>
>>
>>
>> The contents of this electronic message and any attachments are intended
>> only for the addressee and may contain legally privileged, personal,
>> sensitive or confidential information. If you are not the intended
>> addressee, and have received this email, any transmission, distribution,
>> downloading, printing or photocopying of the contents of this message or
>> attachments is strictly prohibited. Any legal privilege or confidentiality
>> attached to this message and attachments is not waived, lost or destroyed by
>> reason of delivery to any person other than intended addressee. If you have
>> received this message and are not the intended addressee you should notify
>> the sender by return email and destroy all copies of the message and any
>> attachments. Unless expressly attributed, the views expressed in this email
>> do not necessarily represent the views of the company.
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
