| Hello got several OSD's that are crashing. Migrated last week from Ubuntu 14.10 (don't know CEPH version) to 14.04 and since then several OSD's are flapping. Here a log file from one of the OSD that is flapping. Would appreciate any help in getting back a stable CEPH cluster. There are 3 Nodes, each with 2 x 3TB SATA and 1 SSD. There are no SMART or SCSI errors visible. Currently the cluster runs ceph version 0.80.7 thanks for any help and cheers marco more logs here: http://pastebin.com/FbfrnLMc -7465> 2015-03-02 06:49:41.259083 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.8c( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/63571) [2,9,0] r=2 lpr=73369 pi=67905-73368/2103 crt=0'0 inactive NOTIFY] exit Start 0.000019 0 0.000000 -7464> 2015-03-02 06:49:41.259093 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.8c( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/63571) [2,9,0] r=2 lpr=73369 pi=67905-73368/2103 crt=0'0 inactive NOTIFY] enter Started/Stray -7463> 2015-03-02 06:49:41.261513 7f5aa2111700 1 -- 192.168.1.22:6805/28026 --> 192.168.1.21:6806/23110 -- pg_query(0.85,1.7e,2.86 epoch 73369) v3 -- ?+0 0x7f5ae7176000 con 0x7f5ae34d8520 -7462> 2015-03-02 06:49:41.266062 7f5aa1910700 5 osd.0 pg_epoch: 73368 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73363/73363/63881) [8,0,2] r=1 lpr=73364 pi=63462-73362/3760 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started/Stray 2.574528 5 0.001934 -7461> 2015-03-02 06:49:41.266064 7f5aa2111700 5 osd.0 pg_epoch: 73368 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73363/73363/63795) [9,6,0] r=2 lpr=73364 pi=67905-73362/2103 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started/Stray 2.568400 5 0.001944 -7460> 2015-03-02 06:49:41.266082 7f5aa1910700 5 osd.0 pg_epoch: 73368 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73363/73363/63881) [8,0,2] r=1 lpr=73364 pi=63462-73362/3760 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started 2.574571 0 0.000000 -7459> 2015-03-02 06:49:41.266082 7f5aa2111700 5 osd.0 pg_epoch: 73368 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73363/73363/63795) [9,6,0] r=2 lpr=73364 pi=67905-73362/2103 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started 2.568449 0 0.000000 -7458> 2015-03-02 06:49:41.266088 7f5aa1910700 5 osd.0 pg_epoch: 73368 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73363/73363/63881) [8,0,2] r=1 lpr=73364 pi=63462-73362/3760 crt=0'0 lcod 0'0 inactive NOTIFY] enter Reset -7457> 2015-03-02 06:49:41.266088 7f5aa2111700 5 osd.0 pg_epoch: 73368 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73363/73363/63795) [9,6,0] r=2 lpr=73364 pi=67905-73362/2103 crt=0'0 lcod 0'0 inactive NOTIFY] enter Reset -7456> 2015-03-02 06:49:41.267053 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73369/73369/63795) [9,6,0] r=2 lpr=73369 pi=67905-73368/2105 crt=0'0 lcod 0'0 inactive NOTIFY] exit Reset 0.000964 2 0.000074 -7455> 2015-03-02 06:49:41.267065 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73369/73369/63795) [9,6,0] r=2 lpr=73369 pi=67905-73368/2105 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started -7454> 2015-03-02 06:49:41.267070 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73369/73369/63795) [9,6,0] r=2 lpr=73369 pi=67905-73368/2105 crt=0'0 lcod 0'0 inactive NOTIFY] enter Start -7453> 2015-03-02 06:49:41.267073 7f5aa2111700 1 osd.0 pg_epoch: 73369 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73369/73369/63795) [9,6,0] r=2 lpr=73369 pi=67905-73368/2105 crt=0'0 lcod 0'0 inactive NOTIFY] state<Start>: transitioning to Stray -7452> 2015-03-02 06:49:41.267079 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73369/73369/63795) [9,6,0] r=2 lpr=73369 pi=67905-73368/2105 crt=0'0 lcod 0'0 inactive NOTIFY] exit Start 0.000009 0 0.000000 -7451> 2015-03-02 06:49:41.267084 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[1.92( v 58213'3589 (3524'589,58213'3589] local-les=73364 n=2 ec=1 les/c 73364/73364 73369/73369/63795) [9,6,0] r=2 lpr=73369 pi=67905-73368/2105 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started/Stray -7450> 2015-03-02 06:49:41.268623 7f5aa2111700 5 osd.0 pg_epoch: 73368 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73363/73363/63553) [8,2,0] r=2 lpr=73364 pi=63561-73362/3751 crt=0'0 inactive NOTIFY] exit Started/Stray 2.567176 5 0.003757 -7449> 2015-03-02 06:49:41.268686 7f5aa2111700 5 osd.0 pg_epoch: 73368 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73363/73363/63553) [8,2,0] r=2 lpr=73364 pi=63561-73362/3751 crt=0'0 inactive NOTIFY] exit Started 2.567259 0 0.000000 -7448> 2015-03-02 06:49:41.268693 7f5aa2111700 5 osd.0 pg_epoch: 73368 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73363/73363/63553) [8,2,0] r=2 lpr=73364 pi=63561-73362/3751 crt=0'0 inactive NOTIFY] enter Reset -7447> 2015-03-02 06:49:41.268950 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73369/73369/63881) [8,0,2] r=1 lpr=73369 pi=63462-73368/3762 crt=0'0 lcod 0'0 inactive NOTIFY] exit Reset 0.002862 2 0.000077 -7446> 2015-03-02 06:49:41.268964 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73369/73369/63881) [8,0,2] r=1 lpr=73369 pi=63462-73368/3762 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started -7445> 2015-03-02 06:49:41.268969 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73369/73369/63881) [8,0,2] r=1 lpr=73369 pi=63462-73368/3762 crt=0'0 lcod 0'0 inactive NOTIFY] enter Start -7444> 2015-03-02 06:49:41.268973 7f5aa1910700 1 osd.0 pg_epoch: 73369 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73369/73369/63881) [8,0,2] r=1 lpr=73369 pi=63462-73368/3762 crt=0'0 lcod 0'0 inactive NOTIFY] state<Start>: transitioning to Stray -7443> 2015-03-02 06:49:41.268980 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73369/73369/63881) [8,0,2] r=1 lpr=73369 pi=63462-73368/3762 crt=0'0 lcod 0'0 inactive NOTIFY] exit Start 0.000011 0 0.000000 -7442> 2015-03-02 06:49:41.268987 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[1.8e( v 63491'4663 (3560'1663,63491'4663] local-les=73364 n=1 ec=1 les/c 73364/73364 73369/73369/63881) [8,0,2] r=1 lpr=73369 pi=63462-73368/3762 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started/Stray -7441> 2015-03-02 06:49:41.270231 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73369/73369/63553) [8,2,0] r=2 lpr=73369 pi=63561-73368/3753 crt=0'0 inactive NOTIFY] exit Reset 0.001537 2 0.000107 -7440> 2015-03-02 06:49:41.270240 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73369/73369/63553) [8,2,0] r=2 lpr=73369 pi=63561-73368/3753 crt=0'0 inactive NOTIFY] enter Started -7439> 2015-03-02 06:49:41.270244 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73369/73369/63553) [8,2,0] r=2 lpr=73369 pi=63561-73368/3753 crt=0'0 inactive NOTIFY] enter Start -7438> 2015-03-02 06:49:41.270247 7f5aa2111700 1 osd.0 pg_epoch: 73369 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73369/73369/63553) [8,2,0] r=2 lpr=73369 pi=63561-73368/3753 crt=0'0 inactive NOTIFY] state<Start>: transitioning to Stray -7437> 2015-03-02 06:49:41.270252 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73369/73369/63553) [8,2,0] r=2 lpr=73369 pi=63561-73368/3753 crt=0'0 inactive NOTIFY] exit Start 0.000007 0 0.000000 -7436> 2015-03-02 06:49:41.270255 7f5aa2111700 5 osd.0 pg_epoch: 73369 pg[2.97( empty local-les=73364 n=0 ec=1 les/c 73364/73364 73369/73369/63553) [8,2,0] r=2 lpr=73369 pi=63561-73368/3753 crt=0'0 inactive NOTIFY] enter Started/Stray -7435> 2015-03-02 06:49:41.270546 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] exit Reset 2.684941 6 0.000050 -7434> 2015-03-02 06:49:41.270553 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364^C/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] enter Started -7433> 2015-03-02 06:49:41.270557 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] enter Start -7432> 2015-03-02 06:49:41.270561 7f5aa1910700 1 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] state<Start>: transitioning to Primary -7431> 2015-03-02 06:49:41.270573 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] exit Start 0.000016 0 0.000000 -7430> 2015-03-02 06:49:41.270579 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] enter Started/Primary -7429> 2015-03-02 06:49:41.270584 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 inactive] enter Started/Primary/Peering -7428> 2015-03-02 06:49:41.270590 7f5aa1910700 5 osd.0 pg_epoch: 73369 pg[2.92( empty local-les=73364 n=0 ec=1 les/c 73364/73365 73369/73369/73369) [0,9,6] r=0 lpr=73369 pi=73363-73368/2 crt=0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetInfo -7427> 2015-03-02 06:49:41.270679 7f5aa1910700 5 osd.0 pg_epoch: 73366 pg[1.91( v 61301'4056 (3475'1056,61301'4056] lb 0//0//-1 local-les=0 n=0 ec=1 les/c 73002/73002 73363/73363/63462) [6,8,0] r=2 lpr=73363 pi=67905-73362/3150 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started/Stray 2.578141 3 0.000962 -7426> 2015-03-02 06:49:41.270691 7f5aa1910700 5 osd.0 pg_epoch: 73366 pg[1.91( v 61301'4056 (3475'1056,61301'4056] lb 0//0//-1 local-les=0 n=0 ec=1 les/c 73002/73 root@nuv-vir-kvm-server-6 ~ # tail -n 200 /var/log/ceph/ceph-osd.0.log 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 keyvaluestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.0.log --- end dump of recent events --- 2015-03-02 08:24:08.212310 7fb7dadce700 -1 *** Caught signal (Aborted) ** in thread 7fb7dadce700 ceph version 0.80.7 (6c0127fcb58008793d3c8b62d925bc91963672a3) 1: (()+0x5b889f) [0x7fb7f8c6889f] 2: (()+0x10340) [0x7fb7f7bc3340] 3: (gsignal()+0x39) [0x7fb7f626acc9] 4: (abort()+0x148) [0x7fb7f626e0d8] 5: (__gnu_cxx::__verbose_terminate_handler()+0x155) [0x7fb7f6b756b5] 6: (()+0x5e836) [0x7fb7f6b73836] 7: (()+0x5e863) [0x7fb7f6b73863] 8: (()+0x5eaa2) [0x7fb7f6b73aa2] 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1f2) [0x7fb7f8d4b1e2] 10: (ReplicatedPG::trim_object(hobject_t const&)+0x9e5) [0x7fb7f8ac14c5] 11: (ReplicatedPG::TrimmingObjects::react(ReplicatedPG::SnapTrim const&)+0x501) [0x7fb7f8af20e1] 12: (boost::statechart::simple_state<ReplicatedPG::TrimmingObjects, ReplicatedPG::SnapTrimmer, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xb4) [0x7fb7f8b2a5f4] 13: (boost::statechart::state_machine<ReplicatedPG::SnapTrimmer, ReplicatedPG::NotTrimming, std::allocator<void>, boost::statechart::null_exception_translator>::process_queued_events()+0x127) [0x7fb7f8b115e7] 14: (boost::statechart::state_machine<ReplicatedPG::SnapTrimmer, ReplicatedPG::NotTrimming, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x84) [0x7fb7f8b117b4] 15: (ReplicatedPG::snap_trimmer()+0x5ec) [0x7fb7f8a8dc0c] 16: (OSD::SnapTrimWQ::_process(PG*)+0x1a) [0x7fb7f893ddfa] 17: (ThreadPool::worker(ThreadPool::WorkThread*)+0xaf1) [0x7fb7f8d3c091] 18: (ThreadPool::WorkThread::entry()+0x10) [0x7fb7f8d3cf80] 19: (()+0x8182) [0x7fb7f7bbb182] 20: (clone()+0x6d) [0x7fb7f632e47d] NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. --- begin dump of recent events --- -68> 2015-03-02 08:24:08.132492 7fb7ea2a6700 1 -- 192.168.1.22:6808/21949 --> osd.6 192.168.1.101:6817/8236 -- pg_info(1 pgs e75029:0.1d) v4 -- ?+0 0x7fb82035b480 -67> 2015-03-02 08:24:08.150539 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.1 192.168.1.22:6802/32030 24 ==== MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 ==== 8390309+0+0 (1733576751 0 0) 0x7fb81f80c1c0 con 0x7fb7fa6b49a0 -66> 2015-03-02 08:24:08.150580 7fb7e22e1700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.139265, event: header_read, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -65> 2015-03-02 08:24:08.150601 7fb7e22e1700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.139269, event: throttled, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -64> 2015-03-02 08:24:08.150613 7fb7e22e1700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.150510, event: all_read, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -63> 2015-03-02 08:24:08.150625 7fb7e22e1700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.150575, event: dispatched, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -62> 2015-03-02 08:24:08.150637 7fb7e22e1700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.150637, event: waiting_for_osdmap, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -61> 2015-03-02 08:24:08.154933 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.1 192.168.1.22:6802/32030 25 ==== MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 ==== 8390277+0+0 (2677569573 0 0) 0x7fb81f80c940 con 0x7fb7fa6b49a0 -60> 2015-03-02 08:24:08.154992 7fb7e22e1700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.150680, event: header_read, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -59> 2015-03-02 08:24:08.155008 7fb7e22e1700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.150682, event: throttled, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -58> 2015-03-02 08:24:08.155019 7fb7e22e1700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.154901, event: all_read, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -57> 2015-03-02 08:24:08.155030 7fb7e22e1700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.154984, event: dispatched, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -56> 2015-03-02 08:24:08.155043 7fb7e22e1700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.155042, event: waiting_for_osdmap, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -55> 2015-03-02 08:24:08.155093 7fb7dbdd0700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.155093, event: reached_pg, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -54> 2015-03-02 08:24:08.155224 7fb7dbdd0700 5 -- op tracker -- , seq: 560, time: 2015-03-02 08:24:08.155224, event: done, request: MOSDPGPush(0.5f 75029 [PushOp(b1b8545f/10000056f30.000000ec/head//0, version: 60836'179935, data_included: [0~4194304], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(b1b8545f/10000056f30.000000ec/head//0@60836'179935, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(1d1e545f/10000056e32.00000b4b/head//0, version: 60822'179492, data_included: [0~4194304], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(1d1e545f/10000056e32.00000b4b/head//0@60822'179492, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -53> 2015-03-02 08:24:08.155253 7fb7dbdd0700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.155253, event: reached_pg, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -52> 2015-03-02 08:24:08.155654 7fb7dbdd0700 5 -- op tracker -- , seq: 561, time: 2015-03-02 08:24:08.155654, event: done, request: MOSDPGPush(0.5f 75029 [PushOp(48f3a45f/10000056f2b.00000564/head//0, version: 60836'179931, data_included: [0~4194304], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(48f3a45f/10000056f2b.00000564/head//0@60836'179931, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0, version: 60947'186774, data_included: [0~4194304], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 1, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/614//0@60947'186774, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -51> 2015-03-02 08:24:08.166120 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.1 192.168.1.22:6802/32030 26 ==== MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 ==== 4195297+0+0 (1919988800 0 0) 0x7fb81f80a5a0 con 0x7fb7fa6b49a0 -50> 2015-03-02 08:24:08.166151 7fb7e22e1700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.154949, event: header_read, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -49> 2015-03-02 08:24:08.166161 7fb7e22e1700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.154952, event: throttled, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -48> 2015-03-02 08:24:08.166170 7fb7e22e1700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.164826, event: all_read, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -47> 2015-03-02 08:24:08.166178 7fb7e22e1700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.166147, event: dispatched, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -46> 2015-03-02 08:24:08.166186 7fb7e22e1700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.166186, event: waiting_for_osdmap, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -45> 2015-03-02 08:24:08.166232 7fb7dbdd0700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.166232, event: reached_pg, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -44> 2015-03-02 08:24:08.173622 7fb7dbdd0700 5 -- op tracker -- , seq: 562, time: 2015-03-02 08:24:08.173622, event: done, request: MOSDPGPush(0.5f 75029 [PushOp(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0, version: 61050'188334, data_included: [0~4194304], data_size: 0, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(4834a45f/rb.0.108f.2ae8944a.0000000006ed/head//0@61050'188334, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 -43> 2015-03-02 08:24:08.182990 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.2 192.168.1.101:6801/19222 101 ==== pg_info(1 pgs e75029:0.9c(4387)) v4 ==== 240083+0+0 (4103439616 0 0) 0x7fb821f7efc0 con 0x7fb7fa6b4160 -42> 2015-03-02 08:24:08.183012 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.124715, event: header_read, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -41> 2015-03-02 08:24:08.183018 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.124716, event: throttled, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -40> 2015-03-02 08:24:08.183022 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.173572, event: all_read, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -39> 2015-03-02 08:24:08.183025 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.183008, event: dispatched, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -38> 2015-03-02 08:24:08.183029 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.183029, event: waiting_for_osdmap, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -37> 2015-03-02 08:24:08.183037 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.183037, event: started, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -36> 2015-03-02 08:24:08.183065 7fb7e22e1700 5 -- op tracker -- , seq: 563, time: 2015-03-02 08:24:08.183065, event: done, request: pg_info(1 pgs e75029:0.9c(4387)) v4 -35> 2015-03-02 08:24:08.183081 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[0.9c( v 69491'84610 (60912'81338,69491'84610] lb 76bc659c/10000056e64.00000a21/head//0 local-les=0 n=114 ec=1 les/c 75027/63461 75028/75028/67905) [0,8,2]/[2,1] r=-1 lpr=75028 pi=63460-75027/4334 crt=69491'84610 lcod 0'0 remapped NOTIFY] exit Started/Stray 4.038494 7 0.008148 -34> 2015-03-02 08:24:08.183105 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[0.9c( v 69491'84610 (60912'81338,69491'84610] lb 76bc659c/10000056e64.00000a21/head//0 local-les=0 n=114 ec=1 les/c 75027/63461 75028/75028/67905) [0,8,2]/[2,1] r=-1 lpr=75028 pi=63460-75027/4334 crt=69491'84610 lcod 0'0 remapped NOTIFY] enter Started/ReplicaActive -33> 2015-03-02 08:24:08.183119 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[0.9c( v 69491'84610 (60912'81338,69491'84610] lb 76bc659c/10000056e64.00000a21/head//0 local-les=0 n=114 ec=1 les/c 75027/63461 75028/75028/67905) [0,8,2]/[2,1] r=-1 lpr=75028 pi=63460-75027/4334 crt=69491'84610 lcod 0'0 remapped NOTIFY] enter Started/ReplicaActive/RepNotRecovering -32> 2015-03-02 08:24:08.200271 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.2 192.168.1.101:6801/19222 102 ==== pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 ==== 388748+0+0 (3775696514 0 0) 0x7fb82527e680 con 0x7fb7fa6b4160 -31> 2015-03-02 08:24:08.200301 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.185704, event: header_read, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -30> 2015-03-02 08:24:08.200307 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.185706, event: throttled, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -29> 2015-03-02 08:24:08.200312 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.200258, event: all_read, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -28> 2015-03-02 08:24:08.200317 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.200296, event: dispatched, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -27> 2015-03-02 08:24:08.200322 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.200322, event: waiting_for_osdmap, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -26> 2015-03-02 08:24:08.200331 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.200331, event: started, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -25> 2015-03-02 08:24:08.200399 7fb7e22e1700 5 -- op tracker -- , seq: 564, time: 2015-03-02 08:24:08.200399, event: done, request: pg_notify(0.b7(2679),0.b8(4327),0.bc(1),2.bc(1) epoch 75029) v5 -24> 2015-03-02 08:24:08.203653 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.6 192.168.1.101:6817/8236 118 ==== pg_info(1 pgs e75029:1.97) v4 ==== 714+0+0 (1762570777 0 0) 0x7fb820874380 con 0x7fb7fa6b4000 -23> 2015-03-02 08:24:08.203674 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203416, event: header_read, request: pg_info(1 pgs e75029:1.97) v4 -22> 2015-03-02 08:24:08.203678 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203418, event: throttled, request: pg_info(1 pgs e75029:1.97) v4 -21> 2015-03-02 08:24:08.203679 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203475, event: all_read, request: pg_info(1 pgs e75029:1.97) v4 -20> 2015-03-02 08:24:08.203681 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203671, event: dispatched, request: pg_info(1 pgs e75029:1.97) v4 -19> 2015-03-02 08:24:08.203683 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203683, event: waiting_for_osdmap, request: pg_info(1 pgs e75029:1.97) v4 -18> 2015-03-02 08:24:08.203687 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203687, event: started, request: pg_info(1 pgs e75029:1.97) v4 -17> 2015-03-02 08:24:08.203704 7fb7e22e1700 5 -- op tracker -- , seq: 565, time: 2015-03-02 08:24:08.203703, event: done, request: pg_info(1 pgs e75029:1.97) v4 -16> 2015-03-02 08:24:08.203716 7fb7e22e1700 1 -- 192.168.1.22:6808/21949 <== osd.6 192.168.1.101:6817/8236 119 ==== pg_info(1 pgs e75029:2.a9) v4 ==== 714+0+0 (3782541911 0 0) 0x7fb82527f9c0 con 0x7fb7fa6b4000 -15> 2015-03-02 08:24:08.203720 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203483, event: header_read, request: pg_info(1 pgs e75029:2.a9) v4 -14> 2015-03-02 08:24:08.203722 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203484, event: throttled, request: pg_info(1 pgs e75029:2.a9) v4 -13> 2015-03-02 08:24:08.203723 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203516, event: all_read, request: pg_info(1 pgs e75029:2.a9) v4 -12> 2015-03-02 08:24:08.203725 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203719, event: dispatched, request: pg_info(1 pgs e75029:2.a9) v4 -11> 2015-03-02 08:24:08.203726 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203726, event: waiting_for_osdmap, request: pg_info(1 pgs e75029:2.a9) v4 -10> 2015-03-02 08:24:08.203728 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203728, event: started, request: pg_info(1 pgs e75029:2.a9) v4 -9> 2015-03-02 08:24:08.203736 7fb7e22e1700 5 -- op tracker -- , seq: 566, time: 2015-03-02 08:24:08.203736, event: done, request: pg_info(1 pgs e75029:2.a9) v4 -8> 2015-03-02 08:24:08.203728 7fb7dc5d1700 1 -- 192.168.1.22:6808/21949 --> osd.6 192.168.1.101:6817/8236 -- pg_info(1 pgs e75029:1.97) v4 -- ?+0 0x7fb8237df800 -7> 2015-03-02 08:24:08.203743 7fb7dc5d1700 1 -- 192.168.1.22:6808/21949 --> osd.9 192.168.1.21:6806/23110 -- pg_info(1 pgs e75029:1.97) v4 -- ?+0 0x7fb8237df640 -6> 2015-03-02 08:24:08.204622 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[1.97( v 48273'5108 (3556'2108,48273'5108] local-les=75029 n=2 ec=1 les/c 75029/75022 75028/75028/75028) [0,9,6] r=0 lpr=75028 pi=75019-75027/2 crt=0'0 lcod 0'0 mlcod 0'0 active] exit Started/Primary/Active/Activating 0.721095 8 0.000307 -5> 2015-03-02 08:24:08.204641 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[1.97( v 48273'5108 (3556'2108,48273'5108] local-les=75029 n=2 ec=1 les/c 75029/75022 75028/75028/75028) [0,9,6] r=0 lpr=75028 pi=75019-75027/2 crt=0'0 lcod 0'0 mlcod 0'0 active] enter Started/Primary/Active/Recovered -4> 2015-03-02 08:24:08.204655 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[1.97( v 48273'5108 (3556'2108,48273'5108] local-les=75029 n=2 ec=1 les/c 75029/75022 75028/75028/75028) [0,9,6] r=0 lpr=75028 pi=75019-75027/2 crt=0'0 lcod 0'0 mlcod 0'0 active] exit Started/Primary/Active/Recovered 0.000014 0 0.000000 -3> 2015-03-02 08:24:08.204666 7fb7dc5d1700 5 osd.0 pg_epoch: 75029 pg[1.97( v 48273'5108 (3556'2108,48273'5108] local-les=75029 n=2 ec=1 les/c 75029/75022 75028/75028/75028) [0,9,6] r=0 lpr=75028 pi=75019-75027/2 crt=0'0 lcod 0'0 mlcod 0'0 active] enter Started/Primary/Active/Clean -2> 2015-03-02 08:24:08.204684 7fb7dc5d1700 1 -- 192.168.1.22:6808/21949 --> osd.6 192.168.1.101:6817/8236 -- pg_info(1 pgs e75029:1.97) v4 -- ?+0 0x7fb8237df480 -1> 2015-03-02 08:24:08.204694 7fb7dc5d1700 1 -- 192.168.1.22:6808/21949 --> osd.9 192.168.1.21:6806/23110 -- pg_info(1 pgs e75029:1.97) v4 -- ?+0 0x7fb8237df2c0 0> 2015-03-02 08:24:08.212310 7fb7dadce700 -1 *** Caught signal (Aborted) ** in thread 7fb7dadce700 ceph version 0.80.7 (6c0127fcb58008793d3c8b62d925bc91963672a3) 1: (()+0x5b889f) [0x7fb7f8c6889f] 2: (()+0x10340) [0x7fb7f7bc3340] 3: (gsignal()+0x39) [0x7fb7f626acc9] 4: (abort()+0x148) [0x7fb7f626e0d8] 5: (__gnu_cxx::__verbose_terminate_handler()+0x155) [0x7fb7f6b756b5] 6: (()+0x5e836) [0x7fb7f6b73836] 7: (()+0x5e863) [0x7fb7f6b73863] 8: (()+0x5eaa2) [0x7fb7f6b73aa2] 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1f2) [0x7fb7f8d4b1e2] 10: (ReplicatedPG::trim_object(hobject_t const&)+0x9e5) [0x7fb7f8ac14c5] 11: (ReplicatedPG::TrimmingObjects::react(ReplicatedPG::SnapTrim const&)+0x501) [0x7fb7f8af20e1] 12: (boost::statechart::simple_state<ReplicatedPG::TrimmingObjects, ReplicatedPG::SnapTrimmer, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xb4) [0x7fb7f8b2a5f4] 13: (boost::statechart::state_machine<ReplicatedPG::SnapTrimmer, ReplicatedPG::NotTrimming, std::allocator<void>, boost::statechart::null_exception_translator>::process_queued_events()+0x127) [0x7fb7f8b115e7] 14: (boost::statechart::state_machine<ReplicatedPG::SnapTrimmer, ReplicatedPG::NotTrimming, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x84) [0x7fb7f8b117b4] 15: (ReplicatedPG::snap_trimmer()+0x5ec) [0x7fb7f8a8dc0c] 16: (OSD::SnapTrimWQ::_process(PG*)+0x1a) [0x7fb7f893ddfa] 17: (ThreadPool::worker(ThreadPool::WorkThread*)+0xaf1) [0x7fb7f8d3c091] 18: (ThreadPool::WorkThread::entry()+0x10) [0x7fb7f8d3cf80] 19: (()+0x8182) [0x7fb7f7bbb182] 20: (clone()+0x6d) [0x7fb7f632e47d] NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 keyvaluestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.0.log --- end dump of recent events --- root@nuv-vir-kvm-server-6 ~ # ceph osd out 0 Marco Kuendig / CEO / Founder Marco's Google Hangout Nuvula AG - Hybrid Clouds |
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
