Hi Jeff, Good to know I found this mailing list in the osd Troubleshooting docs https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd/#an-osd-failed so it should probably be changed if this is the wrong mailing list. Also already figured out a workaround using a bigger second drive, If it's still relevant I can still post it there. Cheers, Dominik On Fr, 2022-04-08 at 09:39 -0400, Jeff Layton wrote: > Hi Dominik, > > This mailing list is mostly for the kernel ceph client bits these days. > You probably want to resend this to dev@xxxxxxx or ceph-users@xxxxxxx. > > Cheers, > Jeff > > On Fri, 2022-04-08 at 09:43 +0000, Dominik Bracht wrote: > > Hello everyone, > > > > i have an osd that randomly just crashed and is refusing to come up > > again. I found a similar looking issue > > https://tracker.ceph.com/issues/50656 but the workaround to use the > > bitmap allocator runs into the same issue > > > > Crash Info: > > { > > "crash_id": "2022-04-07T19:26:44.245668Z_7516fd9f-bc84-4cb5-afcd- > > a54c64753631", > > "timestamp": "2022-04-07T19:26:44.245668Z", > > "process_name": "ceph-osd", > > "entity_name": "osd.8", > > "ceph_version": "16.2.7", > > "utsname_hostname": "sinnoh", > > "utsname_sysname": "Linux", > > "utsname_release": "5.13.19-6-pve", > > "utsname_version": "#1 SMP PVE 5.13.19-14 (Thu, 10 Mar 2022 > > 16:24:52 +0100)", > > "utsname_machine": "x86_64", > > "os_name": "Debian GNU/Linux 11 (bullseye)", > > "os_id": "11", > > "os_version_id": "11", > > "os_version": "11 (bullseye)", > > "assert_condition": "r == 0", > > "assert_func": "void > > BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)", > > "assert_file": "./src/os/bluestore/BlueFS.cc", > > "assert_line": 2352, > > "assert_thread_name": "ceph-osd", > > "assert_msg": "./src/os/bluestore/BlueFS.cc: In function 'void > > BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)' thread > > 7ffbced1af00 time 2022-04- > > 07T21:26:44.194523+0200\n./src/os/bluestore/BlueFS.cc: 2352: FAILED > > ceph_assert(r == 0)\n", > > "backtrace": [ > > "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) > > [0x7ffbcf372140]", > > "gsignal()", > > "abort()", > > "(ceph::__ceph_assert_fail(char const*, char const*, int, > > char > > const*)+0x16e) [0x55716b2bfb30]", > > "/usr/bin/ceph-osd(+0xabcc71) [0x55716b2bfc71]", > > > > "(BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)+0x1a13) > > [0x55716b9ba243]", > > "(BlueFS::_flush(BlueFS::FileWriter*, bool, > > std::unique_lock<std::mutex>&)+0x67) [0x55716b9ba497]", > > "(BlueRocksWritableFile::Append(rocksdb::Slice const&)+0x100) > > [0x55716b9d27d0]", > > "(rocksdb::LegacyWritableFileWrapper::Append(rocksdb::Slice > > const&, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x48) > > [0x55716be9924e]", > > "(rocksdb::WritableFileWriter::WriteBuffered(char const*, > > unsigned long)+0x338) [0x55716c073d18]", > > "(rocksdb::WritableFileWriter::Append(rocksdb::Slice > > const&)+0x5d7) [0x55716c07229b]", > > > > "(rocksdb::BlockBasedTableBuilder::WriteRawBlock(rocksdb::Slice > > const&, rocksdb::CompressionType, rocksdb::BlockHandle*, bool)+0x11d) > > [0x55716c23c2d7]", > > "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice > > const&, rocksdb::BlockHandle*, bool)+0x7d0) [0x55716c23c0be]", > > > > "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::BlockBuilder*, > > rocksdb::BlockHandle*, bool)+0x48) [0x55716c23b8da]", > > "(rocksdb::BlockBasedTableBuilder::Flush()+0x9a) > > [0x55716c23b88a]", > > "(rocksdb::BlockBasedTableBuilder::Add(rocksdb::Slice const&, > > rocksdb::Slice const&)+0x197) [0x55716c23b3bf]", > > "(rocksdb::BuildTable(std::__cxx11::basic_string<char, > > std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, > > rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, > > rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, > > rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, > > std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator > > , > > std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, > > std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneItera > > to > > r, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > > > > , rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, > > std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, > > std::default_delete<rocksdb::IntTblPropCollectorFactory> >, > > std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, > > std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > > > const*, > > unsigned int, std::__cxx11::basic_string<char, > > std::char_traits<char>, > > std::allocator<char> > const&, std::vector<unsigned long, > > std::allocator<unsigned long> >, unsigned long, > > rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, > > rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, > > rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, > > rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned > > long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned > > long)+0x782) [0x55716c1be732]", > > "(rocksdb::DBImpl::WriteLevel0TableForRecovery(int, > > rocksdb::ColumnFamilyData*, rocksdb::MemTable*, > > rocksdb::VersionEdit*)+0x5ea) [0x55716bf37226]", > > "(rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, > > std::allocator<unsigned long> > const&, unsigned long*, bool, > > bool*)+0x1ad1) [0x55716bf35e9d]", > > > > "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescripto > > r, > > std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, > > bool, unsigned long*)+0x159e) [0x55716bf333d4]", > > "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, > > std::__cxx11::basic_string<char, std::char_traits<char>, > > std::allocator<char> > const&, > > std::vector<rocksdb::ColumnFamilyDescriptor, > > std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, > > std::vector<rocksdb::ColumnFamilyHandle*, > > std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, > > bool)+0x677) [0x55716bf386cd]", > > "(rocksdb::DB::Open(rocksdb::DBOptions const&, > > std::__cxx11::basic_string<char, std::char_traits<char>, > > std::allocator<char> > const&, > > std::vector<rocksdb::ColumnFamilyDescriptor, > > std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, > > std::vector<rocksdb::ColumnFamilyHandle*, > > std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x52) > > [0x55716bf37aa4]", > > "(RocksDBStore::do_open(std::ostream&, bool, bool, > > std::__cxx11::basic_string<char, std::char_traits<char>, > > std::allocator<char> > const&)+0x10a6) [0x55716be488b6]", > > "(BlueStore::_open_db(bool, bool, bool)+0xa19) > > [0x55716b8c6b19]", > > "(BlueStore::_open_db_and_around(bool, bool)+0x332) > > [0x55716b90bb92]", > > "(BlueStore::_mount()+0x191) [0x55716b90e531]", > > "(OSD::init()+0x58d) [0x55716b3b55ed]", > > "main()", > > "__libc_start_main()", > > "_start()" > > ] > > } >
