Hello all, I've run into some sort of bug with CephFS. Client reads of a particular file return nothing but 40KB of Null bytes. Doing a rados level get of the inode returns the whole file, correctly. Tested via Linux 4.1, 4.2 kernel clients, and the 0.94.3 fuse client. Attached is a dynamic printk debug of the ceph module from the linux 4.2 client while cat'ing the file. My current thought is that there has to be a cache of the object *somewhere* that a 'rados get' bypasses. Even on hosts that have *never* read the file before, it is returning Null bytes from the kernel and fuse mounts. Background: 24x CentOS 7.1 hosts serving up RBD and CephFS with Ceph 0.94.3. CephFS is a EC k=8, m=4 pool with a size 3 writeback cache in front of it. # rados -p cachepool get 10004096b95.00000000 /tmp/kstat-cache # rados -p ec84pool get 10004096b95.00000000 /tmp/kstat-ec # md5sum /tmp/kstat* ddfbe886420a2cb860b46dc70f4f9a0d /tmp/kstat-cache ddfbe886420a2cb860b46dc70f4f9a0d /tmp/kstat-ec # file /tmp/kstat* /tmp/kstat-cache: Perl script, ASCII text executable /tmp/kstat-ec: Perl script, ASCII text executable # md5sum ~daveturner/bin/kstat 1914e941c2ad5245a23e3e1d27cf8fde /homes/daveturner/bin/kstat # file ~daveturner/bin/kstat /homes/daveturner/bin/kstat: data Thoughts? Any more information you need? -- Adam
[975745.247350] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.247362] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.247367] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.247374] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.247378] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.247381] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.252922] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.252934] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.252937] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.252942] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.252945] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.252946] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.256933] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.256942] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.256946] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.256952] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.256956] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.256960] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.260301] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.260309] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.260314] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.260320] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.260325] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.260328] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.264171] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.264183] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.264189] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.264198] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.264203] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.264208] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.266191] ceph: do_getattr inode ffff88011b418888 mask As mode 040755 [975745.266201] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask As) [975745.266206] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975745.266212] ceph: d_revalidate ffff88020808fd80 'mozes' inode ffff8800d18e4d98 offset 0 [975745.266217] ceph: dentry_lease_is_valid - dentry ffff88020808fd80 = 0 [975745.266221] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask Fs) [975745.266224] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975745.266228] ceph: dir_lease_is_valid dir ffff88011b418888 v9 dentry ffff88020808fd80 v9 = 1 [975745.266232] ceph: d_revalidate ffff88020808fd80 valid [975745.266236] ceph: dentry_lru_touch ffff880217097550 ffff88020808fd80 'mozes' (offset 0) [975745.266240] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.266244] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.266247] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.266252] ceph: d_revalidate ffff8802173ab000 '.terminfo' inode (null) offset 0 [975745.266256] ceph: dentry_lease_is_valid - dentry ffff8802173ab000 = 0 [975745.266260] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask Fs) [975745.266263] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.266267] ceph: dir_lease_is_valid dir ffff8800d18e4d98 v10 dentry ffff8802173ab000 v10 = 1 [975745.266270] ceph: d_revalidate ffff8802173ab000 valid [975745.266273] ceph: dentry_lru_touch ffff880217097aa0 ffff8802173ab000 '.terminfo' (offset 0) [975745.268109] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.268114] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.268121] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.268128] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.268132] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.268136] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.271658] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.271668] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.271672] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.271679] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975745.271683] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975745.271686] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.273642] ceph: do_getattr inode ffff88011b418888 mask As mode 040755 [975745.273652] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask As) [975745.273655] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975745.273661] ceph: d_revalidate ffff88020808fd80 'mozes' inode ffff8800d18e4d98 offset 0 [975745.273666] ceph: dentry_lease_is_valid - dentry ffff88020808fd80 = 0 [975745.273670] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask Fs) [975745.273673] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975745.273677] ceph: dir_lease_is_valid dir ffff88011b418888 v9 dentry ffff88020808fd80 v9 = 1 [975745.273681] ceph: d_revalidate ffff88020808fd80 valid [975745.273685] ceph: dentry_lru_touch ffff880217097550 ffff88020808fd80 'mozes' (offset 0) [975745.273689] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975745.273693] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975745.273696] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.273702] ceph: d_revalidate ffff8802173ab000 '.terminfo' inode (null) offset 0 [975745.273705] ceph: dentry_lease_is_valid - dentry ffff8802173ab000 = 0 [975745.273709] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask Fs) [975745.273712] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975745.273716] ceph: dir_lease_is_valid dir ffff8800d18e4d98 v10 dentry ffff8802173ab000 v10 = 1 [975745.273719] ceph: d_revalidate ffff8802173ab000 valid [975745.273722] ceph: dentry_lru_touch ffff880217097aa0 ffff8802173ab000 '.terminfo' (offset 0) [975746.742530] ceph: mdsc delayed_work [975746.742542] ceph: check_delayed_caps [975746.742549] ceph: lookup_mds_session ffff8800a1539800 1 [975746.742556] ceph: mdsc get_session ffff8800a1539800 1 -> 2 [975746.742562] ceph: send_renew_caps to mds0 (up:active) [975746.742570] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975746.742574] ceph: mdsc con_get ffff8800a1539800 ok (3) [975746.742579] ceph: mdsc get_session ffff8800a1539800 3 -> 4 [975746.742583] ceph: mdsc con_get ffff8800a1539800 ok (4) [975746.742589] ceph: add_cap_releases ffff8800a1539800 mds0 extra 680 [975746.742593] ceph: send_cap_releases mds0 [975746.742598] ceph: mdsc put_session ffff8800a1539800 4 -> 3 [975746.742700] ceph: mdsc con_put ffff8800a1539800 (2) [975746.742705] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975746.743738] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975746.743747] ceph: mdsc con_get ffff8800a1539800 ok (3) [975746.743791] ceph: mdsc con_put ffff8800a1539800 (2) [975746.743796] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975746.743807] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975746.743811] ceph: mdsc con_get ffff8800a1539800 ok (3) [975746.743840] ceph: mdsc con_put ffff8800a1539800 (2) [975746.743845] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975746.743854] ceph: handle_session mds0 renewcaps ffff8800a1539800 state open seq 39251 [975746.743861] ceph: renewed_caps mds0 ttl now 4392503901, was fresh, now stale [975746.743876] ceph: mdsc con_put ffff8800a1539800 (1) [975746.743880] ceph: mdsc put_session ffff8800a1539800 2 -> 1 [975751.557899] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975751.557914] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975751.557921] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.557931] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975751.557937] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975751.557941] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.564216] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975751.564229] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975751.564234] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.564243] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975751.564248] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975751.564252] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.570018] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975751.570027] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975751.570032] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.570039] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975751.570043] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975751.570046] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.574079] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975751.574088] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975751.574092] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.574098] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975751.574102] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975751.574105] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.576857] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975751.576866] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975751.576870] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.576876] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975751.576879] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975751.576883] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975751.584058] ceph: do_getattr inode ffff88011b418888 mask As mode 040755 [975751.584068] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask As) [975751.584072] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975751.584079] ceph: d_revalidate ffff88010b0bf600 'daveturner' inode ffff8800258af818 offset 0 [975751.584084] ceph: dentry_lease_is_valid - dentry ffff88010b0bf600 = 0 [975751.584088] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask Fs) [975751.584091] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975751.584095] ceph: dir_lease_is_valid dir ffff88011b418888 v9 dentry ffff88010b0bf600 v9 = 1 [975751.584099] ceph: d_revalidate ffff88010b0bf600 valid [975751.584103] ceph: dentry_lru_touch ffff8801059f52d0 ffff88010b0bf600 'daveturner' (offset 0) [975751.584107] ceph: do_getattr inode ffff8800258af818 mask As mode 040755 [975751.584111] ceph: __ceph_caps_issued_mask ffff8800258af818 cap ffff88021714c400 issued pAsLsXsFs (mask As) [975751.584114] ceph: __touch_cap ffff8800258af818 cap ffff88021714c400 mds0 [975751.584119] ceph: d_revalidate ffff88010b0bf540 'bin' inode ffff8800258afd68 offset 0 [975751.584122] ceph: dentry_lease_is_valid - dentry ffff88010b0bf540 = 0 [975751.584126] ceph: __ceph_caps_issued_mask ffff8800258af818 cap ffff88021714c400 issued pAsLsXsFs (mask Fs) [975751.584129] ceph: __touch_cap ffff8800258af818 cap ffff88021714c400 mds0 [975751.584133] ceph: dir_lease_is_valid dir ffff8800258af818 v1 dentry ffff88010b0bf540 v1 = 1 [975751.584136] ceph: d_revalidate ffff88010b0bf540 valid [975751.584139] ceph: dentry_lru_touch ffff8801059f5320 ffff88010b0bf540 'bin' (offset 0) [975751.584143] ceph: do_getattr inode ffff8800258afd68 mask As mode 040755 [975751.584146] ceph: __ceph_caps_issued_mask ffff8800258afd68 cap ffff88021714c380 issued pAsLsXsFs (mask As) [975751.584150] ceph: __touch_cap ffff8800258afd68 cap ffff88021714c380 mds0 [975751.584154] ceph: d_revalidate ffff88010b0bf840 'kstat' inode ffff8800258ab308 offset 0 [975751.584157] ceph: dentry_lease_is_valid - dentry ffff88010b0bf840 = 0 [975751.584161] ceph: __ceph_caps_issued_mask ffff8800258afd68 cap ffff88021714c380 issued pAsLsXsFs (mask Fs) [975751.584164] ceph: __touch_cap ffff8800258afd68 cap ffff88021714c380 mds0 [975751.584167] ceph: dir_lease_is_valid dir ffff8800258afd68 v1 dentry ffff88010b0bf840 v1 = 1 [975751.584170] ceph: d_revalidate ffff88010b0bf840 valid [975751.584173] ceph: dentry_lru_touch ffff8801059f5370 ffff88010b0bf840 'kstat' (offset 0) [975751.584177] ceph: do_getattr inode ffff8800258ab308 mask As mode 0100755 [975751.584181] ceph: __ceph_caps_issued_mask ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr (mask As) [975751.584184] ceph: __touch_cap ffff8800258ab308 cap ffff88021714c300 mds0 [975751.584191] ceph: open inode ffff8800258ab308 ino 10004096b95.fffffffffffffffe file ffff8800d38cfe00 flags 32768 (32768) [975751.584196] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975751.584200] ceph: open ffff8800258ab308 fmode 1 want pFscr issued pAsLsXsFscr using existing [975751.584204] ceph: init_file ffff8800258ab308 ffff8800d38cfe00 0100755 (regular) [975751.584210] ceph: do_getattr inode ffff8800258ab308 mask pAsLsXsFs mode 0100755 [975751.584214] ceph: __ceph_caps_issued_mask ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr (mask pAsLsXsFs) [975751.584217] ceph: __touch_cap ffff8800258ab308 cap ffff88021714c300 mds0 [975751.584238] ceph: aio_read ffff8800258ab308 10004096b95.fffffffffffffffe 0~4194304 trying to get caps on ffff8800258ab308 [975751.584243] ceph: get_cap_refs ffff8800258ab308 need Fr want Fc [975751.584247] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975751.584252] ceph: get_cap_refs ffff8800258ab308 have pAsLsXsFscr but not Fc (revoking -) [975751.584255] ceph: get_cap_refs ffff8800258ab308 ret 1 got Fcr [975751.584260] ceph: aio_read ffff8800258ab308 10004096b95.fffffffffffffffe 0~4194304 got cap refs on Fcr [975751.584271] ceph: readpages ffff8800258ab308 file ffff8800d38cfe00 nr_pages 10 max 0 [975751.584275] ceph: start_read ffff8800258ab308 nr_pages 10 is 0~40960 [975751.584288] ceph: start_read ffff8800258ab308 adding ffffea000770ff00 idx 0 [975751.584293] ceph: start_read ffff8800258ab308 adding ffffea000717ba80 idx 1 [975751.584297] ceph: start_read ffff8800258ab308 adding ffffea00070c88c0 idx 2 [975751.584301] ceph: start_read ffff8800258ab308 adding ffffea0008213540 idx 3 [975751.584305] ceph: start_read ffff8800258ab308 adding ffffea0007717ec0 idx 4 [975751.584308] ceph: start_read ffff8800258ab308 adding ffffea0007216dc0 idx 5 [975751.584311] ceph: start_read ffff8800258ab308 adding ffffea000845b780 idx 6 [975751.584315] ceph: start_read ffff8800258ab308 adding ffffea00076de6c0 idx 7 [975751.584318] ceph: start_read ffff8800258ab308 adding ffffea00070f72c0 idx 8 [975751.584321] ceph: start_read ffff8800258ab308 adding ffffea0007e50040 idx 9 [975751.584329] ceph: start_read ffff8800258ab308 starting ffff8802146d6900 0~40960 [975751.584403] ceph: readpages ffff8800258ab308 file ffff8800d38cfe00 ret 10 [975751.743245] ceph: mdsc delayed_work [975751.743254] ceph: check_delayed_caps [975751.743262] ceph: lookup_mds_session ffff8800a1539800 1 [975751.743268] ceph: mdsc get_session ffff8800a1539800 1 -> 2 [975751.743275] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975751.743279] ceph: mdsc con_get ffff8800a1539800 ok (3) [975751.743286] ceph: add_cap_releases ffff8800a1539800 mds0 extra 680 [975751.743290] ceph: send_cap_releases mds0 [975751.743294] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975751.743317] ceph: mdsc con_put ffff8800a1539800 (1) [975751.743321] ceph: mdsc put_session ffff8800a1539800 2 -> 1 [975752.825205] ceph: finish_read ffff8800258ab308 req ffff8802146d6900 rc 0 bytes 0 [975752.825219] ceph: finish_read ffff8800258ab308 uptodate ffffea000770ff00 idx 0 [975752.825244] ceph: finish_read ffff8800258ab308 uptodate ffffea000717ba80 idx 1 [975752.825252] ceph: finish_read ffff8800258ab308 uptodate ffffea00070c88c0 idx 2 [975752.825259] ceph: finish_read ffff8800258ab308 uptodate ffffea0008213540 idx 3 [975752.825266] ceph: finish_read ffff8800258ab308 uptodate ffffea0007717ec0 idx 4 [975752.825272] ceph: finish_read ffff8800258ab308 uptodate ffffea0007216dc0 idx 5 [975752.825279] ceph: finish_read ffff8800258ab308 uptodate ffffea000845b780 idx 6 [975752.825286] ceph: finish_read ffff8800258ab308 uptodate ffffea00076de6c0 idx 7 [975752.825292] ceph: finish_read ffff8800258ab308 uptodate ffffea00070f72c0 idx 8 [975752.825299] ceph: finish_read ffff8800258ab308 uptodate ffffea0007e50040 idx 9 [975752.825436] ceph: aio_read ffff8800258ab308 10004096b95.fffffffffffffffe dropping cap refs on Fcr = 40602 [975752.825448] ceph: put_cap_refs ffff8800258ab308 had Fcr last [975752.825458] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975752.825468] ceph: check_caps ffff8800258ab308 file_want pFscr used Fc dirty - flushing - issued pAsLsXsFscr revoking - retain pAsxLsxXsxFsxcrwbl [975752.825476] ceph: mds0 cap ffff88021714c300 used pAsLsXsFscr issued Fc implemented pAsLsXsFscr revoking - [975752.825485] ceph: __send_cap ffff8800258ab308 cap ffff88021714c300 session ffff8800a1539800 pAsLsXsFscr -> pAsLsXsFscr (revoking -) [975752.825497] ceph: send_cap_msg update a8a86b 10004096b95 caps pAsLsXsFscr wanted pFscr dirty - seq 1/1 mseq 0 follows 0 size 40602/0 xattr_ver 0 xattr_len 0 [975752.825506] ceph: mdsc get_session ffff8800a1539800 1 -> 2 [975752.825510] ceph: mdsc con_get ffff8800a1539800 ok (2) [975752.825514] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975752.825518] ceph: mdsc con_get ffff8800a1539800 ok (3) [975752.825532] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975752.825540] ceph: check_caps ffff8800258ab308 file_want pFscr used Fc dirty - flushing - issued pAsLsXsFscr revoking - retain pAsxLsxXsxFsxcrwbl [975752.825545] ceph: __cap_delay_cancel ffff8800258ab308 [975752.825758] ceph: mdsc con_put ffff8800a1539800 (2) [975752.825764] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975752.826570] ceph: aio_read ffff8800258ab308 10004096b95.fffffffffffffffe 40602~4194304 trying to get caps on ffff8800258ab308 [975752.826583] ceph: get_cap_refs ffff8800258ab308 need Fr want Fc [975752.826592] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975752.826599] ceph: get_cap_refs ffff8800258ab308 have pAsLsXsFscr but not Fc (revoking -) [975752.826605] ceph: get_cap_refs ffff8800258ab308 ret 1 got Fcr [975752.826613] ceph: aio_read ffff8800258ab308 10004096b95.fffffffffffffffe 40602~4194304 got cap refs on Fcr [975752.826622] ceph: aio_read ffff8800258ab308 10004096b95.fffffffffffffffe dropping cap refs on Fcr = 0 [975752.826627] ceph: put_cap_refs ffff8800258ab308 had Fcr last [975752.826633] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975752.826643] ceph: check_caps ffff8800258ab308 file_want pFscr used Fc dirty - flushing - issued pAsLsXsFscr revoking - retain pAsxLsxXsxFsxcrwbl [975752.826650] ceph: mds0 cap ffff88021714c300 used pAsLsXsFscr issued Fc implemented pAsLsXsFscr revoking - [975752.826655] ceph: __cap_delay_cancel ffff8800258ab308 [975752.826716] ceph: release inode ffff8800258ab308 file ffff8800d38cfe00 [975752.826721] ceph: put_fmode ffff8800258ab308 fmode 1 1 -> 0 [975752.826728] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975752.826736] ceph: check_caps ffff8800258ab308 file_want - used Fc dirty - flushing - issued pAsLsXsFscr revoking - retain pAsxLsxXsxFsxcrwbl [975752.826741] ceph: check_caps trying to invalidate on ffff8800258ab308 [975752.826759] ceph: try_nonblocking_invalidate ffff8800258ab308 failed [975752.826763] ceph: check_caps failed to invalidate pages [975752.826768] ceph: __cap_set_timeouts ffff8800258ab308 min 482 max 5982 [975752.826773] ceph: __ceph_caps_issued ffff8800258ab308 cap ffff88021714c300 issued pAsLsXsFscr [975752.826785] ceph: check_caps ffff8800258ab308 file_want - used Fc dirty - flushing - issued pAsLsXsFscr revoking - retain pAsxLsxXsxFsxcrwbl [975752.826791] ceph: mds0 cap ffff88021714c300 used pAsLsXsFscr issued Fc implemented pAsLsXsFscr revoking - [975752.826797] ceph: delaying issued pAsLsXsFscr -> pAsLsXsFscr, wanted pFscr -> Fc [975752.826801] ceph: __cap_set_timeouts ffff8800258ab308 min 482 max 5982 [975752.826806] ceph: __cap_delay_requeue ffff8800258ab308 flags 0 at 4392504491 [975752.830884] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.830893] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.830900] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.830907] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.830910] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.830913] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.837357] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.837370] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.837375] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.837383] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.837389] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.837393] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.841185] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.841195] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.841200] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.841208] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.841214] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.841218] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.844739] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.844746] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.844752] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.844759] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.844763] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.844768] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.850094] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.850104] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.850109] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.850116] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.850121] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.850124] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.851792] ceph: do_getattr inode ffff88011b418888 mask As mode 040755 [975752.851801] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask As) [975752.851806] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975752.851812] ceph: d_revalidate ffff88020808fd80 'mozes' inode ffff8800d18e4d98 offset 0 [975752.851816] ceph: dentry_lease_is_valid - dentry ffff88020808fd80 = 0 [975752.851821] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask Fs) [975752.851824] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975752.851828] ceph: dir_lease_is_valid dir ffff88011b418888 v9 dentry ffff88020808fd80 v9 = 1 [975752.851832] ceph: d_revalidate ffff88020808fd80 valid [975752.851836] ceph: dentry_lru_touch ffff880217097550 ffff88020808fd80 'mozes' (offset 0) [975752.851840] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.851844] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.851847] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.851852] ceph: d_revalidate ffff8802173ab000 '.terminfo' inode (null) offset 0 [975752.851856] ceph: dentry_lease_is_valid - dentry ffff8802173ab000 = 0 [975752.851860] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask Fs) [975752.851863] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.851867] ceph: dir_lease_is_valid dir ffff8800d18e4d98 v10 dentry ffff8802173ab000 v10 = 1 [975752.851870] ceph: d_revalidate ffff8802173ab000 valid [975752.851873] ceph: dentry_lru_touch ffff880217097aa0 ffff8802173ab000 '.terminfo' (offset 0) [975752.853745] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.853751] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.853754] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.853758] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.853761] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.853766] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.857247] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.857256] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.857260] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.857266] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975752.857270] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975752.857273] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.858751] ceph: do_getattr inode ffff88011b418888 mask As mode 040755 [975752.858757] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask As) [975752.858760] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975752.858763] ceph: d_revalidate ffff88020808fd80 'mozes' inode ffff8800d18e4d98 offset 0 [975752.858765] ceph: dentry_lease_is_valid - dentry ffff88020808fd80 = 0 [975752.858768] ceph: __ceph_caps_issued_mask ffff88011b418888 cap ffff880217143d00 issued pAsLsXsFs (mask Fs) [975752.858770] ceph: __touch_cap ffff88011b418888 cap ffff880217143d00 mds0 [975752.858773] ceph: dir_lease_is_valid dir ffff88011b418888 v9 dentry ffff88020808fd80 v9 = 1 [975752.858775] ceph: d_revalidate ffff88020808fd80 valid [975752.858778] ceph: dentry_lru_touch ffff880217097550 ffff88020808fd80 'mozes' (offset 0) [975752.858781] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975752.858783] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975752.858785] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.858788] ceph: d_revalidate ffff8802173ab000 '.terminfo' inode (null) offset 0 [975752.858790] ceph: dentry_lease_is_valid - dentry ffff8802173ab000 = 0 [975752.858793] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask Fs) [975752.858795] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975752.858798] ceph: dir_lease_is_valid dir ffff8800d18e4d98 v10 dentry ffff8802173ab000 v10 = 1 [975752.858800] ceph: d_revalidate ffff8802173ab000 valid [975752.858802] ceph: dentry_lru_touch ffff880217097aa0 ffff8802173ab000 '.terminfo' (offset 0) [975753.032666] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975753.032675] ceph: mdsc con_get ffff8800a1539800 ok (3) [975753.032724] ceph: mdsc con_put ffff8800a1539800 (2) [975753.032729] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975753.032738] ceph: mdsc con_put ffff8800a1539800 (1) [975753.032743] ceph: mdsc put_session ffff8800a1539800 2 -> 1 [975756.743954] ceph: mdsc delayed_work [975756.743965] ceph: check_delayed_caps [975756.743973] ceph: lookup_mds_session ffff8800a1539800 1 [975756.743979] ceph: mdsc get_session ffff8800a1539800 1 -> 2 [975756.743986] ceph: mdsc get_session ffff8800a1539800 2 -> 3 [975756.743990] ceph: mdsc con_get ffff8800a1539800 ok (3) [975756.743997] ceph: add_cap_releases ffff8800a1539800 mds0 extra 680 [975756.744001] ceph: send_cap_releases mds0 [975756.744005] ceph: mdsc put_session ffff8800a1539800 3 -> 2 [975756.744068] ceph: mdsc con_put ffff8800a1539800 (1) [975756.744073] ceph: mdsc put_session ffff8800a1539800 2 -> 1 [975760.999937] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975760.999954] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975760.999961] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975760.999971] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975760.999977] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975760.999982] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.005818] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975761.005877] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975761.005883] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.005894] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975761.005899] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975761.005902] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.011819] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975761.011827] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975761.011831] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.011837] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975761.011840] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975761.011844] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.016085] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975761.016095] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975761.016100] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.016107] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975761.016112] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975761.016116] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.019485] ceph: do_getattr inode ffff8800d18e4d98 mask As mode 040755 [975761.019493] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask As) [975761.019497] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0 [975761.019503] ceph: do_getattr inode ffff8800d18e4d98 mask pAsLsXsFs mode 040755 [975761.019507] ceph: __ceph_caps_issued_mask ffff8800d18e4d98 cap ffff880217144800 issued pAsLsXsFs (mask pAsLsXsFs) [975761.019511] ceph: __touch_cap ffff8800d18e4d98 cap ffff880217144800 mds0
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
