Hello everyone, i have an osd that randomly just crashed and is refusing to come up again. I found a similar looking issue https://tracker.ceph.com/issues/50656 but the workaround to use the bitmap allocator runs into the same issue Crash Info: { "crash_id": "2022-04-07T19:26:44.245668Z_7516fd9f-bc84-4cb5-afcd- a54c64753631", "timestamp": "2022-04-07T19:26:44.245668Z", "process_name": "ceph-osd", "entity_name": "osd.8", "ceph_version": "16.2.7", "utsname_hostname": "sinnoh", "utsname_sysname": "Linux", "utsname_release": "5.13.19-6-pve", "utsname_version": "#1 SMP PVE 5.13.19-14 (Thu, 10 Mar 2022 16:24:52 +0100)", "utsname_machine": "x86_64", "os_name": "Debian GNU/Linux 11 (bullseye)", "os_id": "11", "os_version_id": "11", "os_version": "11 (bullseye)", "assert_condition": "r == 0", "assert_func": "void BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)", "assert_file": "./src/os/bluestore/BlueFS.cc", "assert_line": 2352, "assert_thread_name": "ceph-osd", "assert_msg": "./src/os/bluestore/BlueFS.cc: In function 'void BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)' thread 7ffbced1af00 time 2022-04- 07T21:26:44.194523+0200\n./src/os/bluestore/BlueFS.cc: 2352: FAILED ceph_assert(r == 0)\n", "backtrace": [ "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7ffbcf372140]", "gsignal()", "abort()", "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x16e) [0x55716b2bfb30]", "/usr/bin/ceph-osd(+0xabcc71) [0x55716b2bfc71]", "(BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)+0x1a13) [0x55716b9ba243]", "(BlueFS::_flush(BlueFS::FileWriter*, bool, std::unique_lock<std::mutex>&)+0x67) [0x55716b9ba497]", "(BlueRocksWritableFile::Append(rocksdb::Slice const&)+0x100) [0x55716b9d27d0]", "(rocksdb::LegacyWritableFileWrapper::Append(rocksdb::Slice const&, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x48) [0x55716be9924e]", "(rocksdb::WritableFileWriter::WriteBuffered(char const*, unsigned long)+0x338) [0x55716c073d18]", "(rocksdb::WritableFileWriter::Append(rocksdb::Slice const&)+0x5d7) [0x55716c07229b]", "(rocksdb::BlockBasedTableBuilder::WriteRawBlock(rocksdb::Slice const&, rocksdb::CompressionType, rocksdb::BlockHandle*, bool)+0x11d) [0x55716c23c2d7]", "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*, bool)+0x7d0) [0x55716c23c0be]", "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::BlockBuilder*, rocksdb::BlockHandle*, bool)+0x48) [0x55716c23b8da]", "(rocksdb::BlockBasedTableBuilder::Flush()+0x9a) [0x55716c23b88a]", "(rocksdb::BlockBasedTableBuilder::Add(rocksdb::Slice const&, rocksdb::Slice const&)+0x197) [0x55716c23b3bf]", "(rocksdb::BuildTable(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterato r, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > >, rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> >, std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<unsigned long, std::allocator<unsigned long> >, unsigned long, rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned long)+0x782) [0x55716c1be732]", "(rocksdb::DBImpl::WriteLevel0TableForRecovery(int, rocksdb::ColumnFamilyData*, rocksdb::MemTable*, rocksdb::VersionEdit*)+0x5ea) [0x55716bf37226]", "(rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long*, bool, bool*)+0x1ad1) [0x55716bf35e9d]", "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, bool, unsigned long*)+0x159e) [0x55716bf333d4]", "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, bool)+0x677) [0x55716bf386cd]", "(rocksdb::DB::Open(rocksdb::DBOptions const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x52) [0x55716bf37aa4]", "(RocksDBStore::do_open(std::ostream&, bool, bool, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)+0x10a6) [0x55716be488b6]", "(BlueStore::_open_db(bool, bool, bool)+0xa19) [0x55716b8c6b19]", "(BlueStore::_open_db_and_around(bool, bool)+0x332) [0x55716b90bb92]", "(BlueStore::_mount()+0x191) [0x55716b90e531]", "(OSD::init()+0x58d) [0x55716b3b55ed]", "main()", "__libc_start_main()", "_start()" ] }
