I have a problem with one osd (osd.5 on server lod) that keeps crashing.
Often it immediately crashes on restart, but oddly a server reboot fixes
that, also it alwats starts ok from the command line. Service status and
journalctl don't show any useful information.
There's two osd's on the server, the other osd never has a problem.
Server
* osd services only
* 8GB Ram
* Nautilus 14.2.9
* osd.5 : 1TB - crashes
* osd.12 : 500GB - Fine
So I ran it from the command line and copied the console dump when it
crashed. Any thoughts? should I create a bug report for it?
--
Lindsay
root@lod:~# ceph-osd -f --cluster ceph --id 5 --setuser ceph --setgroup ceph
2020-06-22 09:14:51.578 7f8697d16c80 -1 osd.5 7746 log_to_monitors {default=true}
2020-06-22 09:14:51.590 7f8691022700 -1 osd.5 7746 set_numa_affinity unable to identify public interface 'data' numa node: (2) No such file or directory
/build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: In function 'BlueStore::ExtentMap::fault_range(KeyValueDB*, uint32_t, uint32_t)::<lambda(const string&)>' thread 7f867d7ea700 time 2020-06-22 09:16:46.557496
/build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: 2841: FAILED ceph_assert(r >= 0)
2020-06-22 09:16:46.551 7f867d7ea700 -1 bluestore.extentmap(0x5635271bac50) operator() missing shard 0x0 for #2:8c8e1baa:::rbd_data.14c0d3168cc1ea.00000000000044b1:head#
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x563506ff4f1a]
2: (()+0x5050f2) [0x563506ff50f2]
3: (()+0xa1f984) [0x56350750f984]
4: (()+0xa0cea1) [0x5635074fcea1]
5: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
6: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
7: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
8: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
9: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
10: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
11: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
12: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
13: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
14: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
15: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
16: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
17: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
18: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
19: (()+0x7fa3) [0x7f869875afa3]
20: (clone()+0x3f) [0x7f869830a4cf]
*** Caught signal (Aborted) **
in thread 7f867d7ea700 thread_name:tp_osd_tp
2020-06-22 09:16:46.559 7f867d7ea700 -1 /build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: In function 'BlueStore::ExtentMap::fault_range(KeyValueDB*, uint32_t, uint32_t)::<lambda(const string&)>' thread 7f867d7ea700 time 2020-06-22 09:16:46.557496
/build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: 2841: FAILED ceph_assert(r >= 0)
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x563506ff4f1a]
2: (()+0x5050f2) [0x563506ff50f2]
3: (()+0xa1f984) [0x56350750f984]
4: (()+0xa0cea1) [0x5635074fcea1]
5: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
6: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
7: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
8: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
9: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
10: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
11: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
12: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
13: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
14: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
15: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
16: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
17: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
18: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
19: (()+0x7fa3) [0x7f869875afa3]
20: (clone()+0x3f) [0x7f869830a4cf]
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (()+0x12730) [0x7f8698765730]
2: (gsignal()+0x10b) [0x7f86982487bb]
3: (abort()+0x121) [0x7f8698233535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x563506ff4f6b]
5: (()+0x5050f2) [0x563506ff50f2]
6: (()+0xa1f984) [0x56350750f984]
7: (()+0xa0cea1) [0x5635074fcea1]
8: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
9: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
10: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
11: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
12: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
13: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
14: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
15: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
16: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
17: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
18: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
19: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
20: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
21: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
22: (()+0x7fa3) [0x7f869875afa3]
23: (clone()+0x3f) [0x7f869830a4cf]
2020-06-22 09:16:46.563 7f867d7ea700 -1 *** Caught signal (Aborted) **
in thread 7f867d7ea700 thread_name:tp_osd_tp
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (()+0x12730) [0x7f8698765730]
2: (gsignal()+0x10b) [0x7f86982487bb]
3: (abort()+0x121) [0x7f8698233535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x563506ff4f6b]
5: (()+0x5050f2) [0x563506ff50f2]
6: (()+0xa1f984) [0x56350750f984]
7: (()+0xa0cea1) [0x5635074fcea1]
8: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
9: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
10: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
11: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
12: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
13: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
14: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
15: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
16: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
17: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
18: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
19: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
20: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
21: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
22: (()+0x7fa3) [0x7f869875afa3]
23: (clone()+0x3f) [0x7f869830a4cf]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
-151> 2020-06-22 09:14:51.578 7f8697d16c80 -1 osd.5 7746 log_to_monitors {default=true}
-144> 2020-06-22 09:14:51.590 7f8691022700 -1 osd.5 7746 set_numa_affinity unable to identify public interface 'data' numa node: (2) No such file or directory
-2> 2020-06-22 09:16:46.551 7f867d7ea700 -1 bluestore.extentmap(0x5635271bac50) operator() missing shard 0x0 for #2:8c8e1baa:::rbd_data.14c0d3168cc1ea.00000000000044b1:head#
-1> 2020-06-22 09:16:46.559 7f867d7ea700 -1 /build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: In function 'BlueStore::ExtentMap::fault_range(KeyValueDB*, uint32_t, uint32_t)::<lambda(const string&)>' thread 7f867d7ea700 time 2020-06-22 09:16:46.557496
/build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: 2841: FAILED ceph_assert(r >= 0)
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x563506ff4f1a]
2: (()+0x5050f2) [0x563506ff50f2]
3: (()+0xa1f984) [0x56350750f984]
4: (()+0xa0cea1) [0x5635074fcea1]
5: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
6: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
7: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
8: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
9: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
10: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
11: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
12: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
13: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
14: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
15: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
16: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
17: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
18: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
19: (()+0x7fa3) [0x7f869875afa3]
20: (clone()+0x3f) [0x7f869830a4cf]
0> 2020-06-22 09:16:46.563 7f867d7ea700 -1 *** Caught signal (Aborted) **
in thread 7f867d7ea700 thread_name:tp_osd_tp
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (()+0x12730) [0x7f8698765730]
2: (gsignal()+0x10b) [0x7f86982487bb]
3: (abort()+0x121) [0x7f8698233535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x563506ff4f6b]
5: (()+0x5050f2) [0x563506ff50f2]
6: (()+0xa1f984) [0x56350750f984]
7: (()+0xa0cea1) [0x5635074fcea1]
8: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
9: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
10: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
11: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
12: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
13: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
14: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
15: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
16: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
17: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
18: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
19: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
20: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
21: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
22: (()+0x7fa3) [0x7f869875afa3]
23: (clone()+0x3f) [0x7f869830a4cf]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
-151> 2020-06-22 09:14:51.578 7f8697d16c80 -1 osd.5 7746 log_to_monitors {default=true}
-144> 2020-06-22 09:14:51.590 7f8691022700 -1 osd.5 7746 set_numa_affinity unable to identify public interface 'data' numa node: (2) No such file or directory
-2> 2020-06-22 09:16:46.551 7f867d7ea700 -1 bluestore.extentmap(0x5635271bac50) operator() missing shard 0x0 for #2:8c8e1baa:::rbd_data.14c0d3168cc1ea.00000000000044b1:head#
-1> 2020-06-22 09:16:46.559 7f867d7ea700 -1 /build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: In function 'BlueStore::ExtentMap::fault_range(KeyValueDB*, uint32_t, uint32_t)::<lambda(const string&)>' thread 7f867d7ea700 time 2020-06-22 09:16:46.557496
/build/ceph/ceph-14.2.9/src/os/bluestore/BlueStore.cc: 2841: FAILED ceph_assert(r >= 0)
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x563506ff4f1a]
2: (()+0x5050f2) [0x563506ff50f2]
3: (()+0xa1f984) [0x56350750f984]
4: (()+0xa0cea1) [0x5635074fcea1]
5: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
6: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
7: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
8: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
9: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
10: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
11: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
12: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
13: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
14: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
15: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
16: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
17: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
18: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
19: (()+0x7fa3) [0x7f869875afa3]
20: (clone()+0x3f) [0x7f869830a4cf]
0> 2020-06-22 09:16:46.563 7f867d7ea700 -1 *** Caught signal (Aborted) **
in thread 7f867d7ea700 thread_name:tp_osd_tp
ceph version 14.2.9 (bed944f8c45b9c98485e99b70e11bbcec6f6659a) nautilus (stable)
1: (()+0x12730) [0x7f8698765730]
2: (gsignal()+0x10b) [0x7f86982487bb]
3: (abort()+0x121) [0x7f8698233535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x563506ff4f6b]
5: (()+0x5050f2) [0x563506ff50f2]
6: (()+0xa1f984) [0x56350750f984]
7: (()+0xa0cea1) [0x5635074fcea1]
8: (BlueStore::ExtentMap::fault_range(KeyValueDB*, unsigned int, unsigned int)+0x557) [0x5635075152e7]
9: (BlueStore::_do_truncate(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, std::set<BlueStore::SharedBlob*, std::less<BlueStore::SharedBlob*>, std::allocator<BlueStore::SharedBlob*> >*)+0x33f) [0x56350756bd9f]
10: (BlueStore::_do_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>)+0xd1) [0x56350756c5c1]
11: (BlueStore::_remove(BlueStore::TransContext*, boost::intrusive_ptr<BlueStore::Collection>&, boost::intrusive_ptr<BlueStore::Onode>&)+0x62) [0x56350756dfe2]
12: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ObjectStore::Transaction*)+0x153d) [0x563507572f2d]
13: (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x3c8) [0x563507574a98]
14: (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x80) [0x563507158ab0]
15: (PrimaryLogPG::do_backfill_remove(boost::intrusive_ptr<OpRequest>)+0x1f0) [0x563507238150]
16: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x76b) [0x5635072a9f8b]
17: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x397) [0x5635070d6f37]
18: (PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x56350737aa72]
19: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x7d7) [0x5635070f24c7]
20: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b4) [0x5635076b0d04]
21: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x5635076b3710]
22: (()+0x7fa3) [0x7f869875afa3]
23: (clone()+0x3f) [0x7f869830a4cf]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Aborted
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
