Hi Dominik, This mailing list is mostly for the kernel ceph client bits these days. You probably want to resend this to dev@xxxxxxx or ceph-users@xxxxxxx. Cheers, Jeff On Fri, 2022-04-08 at 09:43 +0000, Dominik Bracht wrote: > Hello everyone, > > i have an osd that randomly just crashed and is refusing to come up > again. I found a similar looking issue > https://tracker.ceph.com/issues/50656 but the workaround to use the > bitmap allocator runs into the same issue > > Crash Info: > { > "crash_id": "2022-04-07T19:26:44.245668Z_7516fd9f-bc84-4cb5-afcd- > a54c64753631", > "timestamp": "2022-04-07T19:26:44.245668Z", > "process_name": "ceph-osd", > "entity_name": "osd.8", > "ceph_version": "16.2.7", > "utsname_hostname": "sinnoh", > "utsname_sysname": "Linux", > "utsname_release": "5.13.19-6-pve", > "utsname_version": "#1 SMP PVE 5.13.19-14 (Thu, 10 Mar 2022 > 16:24:52 +0100)", > "utsname_machine": "x86_64", > "os_name": "Debian GNU/Linux 11 (bullseye)", > "os_id": "11", > "os_version_id": "11", > "os_version": "11 (bullseye)", > "assert_condition": "r == 0", > "assert_func": "void > BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)", > "assert_file": "./src/os/bluestore/BlueFS.cc", > "assert_line": 2352, > "assert_thread_name": "ceph-osd", > "assert_msg": "./src/os/bluestore/BlueFS.cc: In function 'void > BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)' thread > 7ffbced1af00 time 2022-04- > 07T21:26:44.194523+0200\n./src/os/bluestore/BlueFS.cc: 2352: FAILED > ceph_assert(r == 0)\n", > "backtrace": [ > "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) > [0x7ffbcf372140]", > "gsignal()", > "abort()", > "(ceph::__ceph_assert_fail(char const*, char const*, int, char > const*)+0x16e) [0x55716b2bfb30]", > "/usr/bin/ceph-osd(+0xabcc71) [0x55716b2bfc71]", > > "(BlueFS::_compact_log_async(std::unique_lock<std::mutex>&)+0x1a13) > [0x55716b9ba243]", > "(BlueFS::_flush(BlueFS::FileWriter*, bool, > std::unique_lock<std::mutex>&)+0x67) [0x55716b9ba497]", > "(BlueRocksWritableFile::Append(rocksdb::Slice const&)+0x100) > [0x55716b9d27d0]", > "(rocksdb::LegacyWritableFileWrapper::Append(rocksdb::Slice > const&, rocksdb::IOOptions const&, rocksdb::IODebugContext*)+0x48) > [0x55716be9924e]", > "(rocksdb::WritableFileWriter::WriteBuffered(char const*, > unsigned long)+0x338) [0x55716c073d18]", > "(rocksdb::WritableFileWriter::Append(rocksdb::Slice > const&)+0x5d7) [0x55716c07229b]", > "(rocksdb::BlockBasedTableBuilder::WriteRawBlock(rocksdb::Slice > const&, rocksdb::CompressionType, rocksdb::BlockHandle*, bool)+0x11d) > [0x55716c23c2d7]", > "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice > const&, rocksdb::BlockHandle*, bool)+0x7d0) [0x55716c23c0be]", > > "(rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::BlockBuilder*, > rocksdb::BlockHandle*, bool)+0x48) [0x55716c23b8da]", > "(rocksdb::BlockBasedTableBuilder::Flush()+0x9a) > [0x55716c23b88a]", > "(rocksdb::BlockBasedTableBuilder::Add(rocksdb::Slice const&, > rocksdb::Slice const&)+0x197) [0x55716c23b3bf]", > "(rocksdb::BuildTable(std::__cxx11::basic_string<char, > std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, > rocksdb::FileSystem*, rocksdb::ImmutableCFOptions const&, > rocksdb::MutableCFOptions const&, rocksdb::FileOptions const&, > rocksdb::TableCache*, rocksdb::InternalIteratorBase<rocksdb::Slice>*, > std::vector<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterator, > std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> >, > std::allocator<std::unique_ptr<rocksdb::FragmentedRangeTombstoneIterato > r, std::default_delete<rocksdb::FragmentedRangeTombstoneIterator> > > > > , rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&, > std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, > std::default_delete<rocksdb::IntTblPropCollectorFactory> >, > std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory, > std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*, > unsigned int, std::__cxx11::basic_string<char, std::char_traits<char>, > std::allocator<char> > const&, std::vector<unsigned long, > std::allocator<unsigned long> >, unsigned long, > rocksdb::SnapshotChecker*, rocksdb::CompressionType, unsigned long, > rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*, > rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int, > rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned > long, unsigned long, rocksdb::Env::WriteLifeTimeHint, unsigned > long)+0x782) [0x55716c1be732]", > "(rocksdb::DBImpl::WriteLevel0TableForRecovery(int, > rocksdb::ColumnFamilyData*, rocksdb::MemTable*, > rocksdb::VersionEdit*)+0x5ea) [0x55716bf37226]", > "(rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long, > std::allocator<unsigned long> > const&, unsigned long*, bool, > bool*)+0x1ad1) [0x55716bf35e9d]", > > "(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor, > std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool, > bool, unsigned long*)+0x159e) [0x55716bf333d4]", > "(rocksdb::DBImpl::Open(rocksdb::DBOptions const&, > std::__cxx11::basic_string<char, std::char_traits<char>, > std::allocator<char> > const&, > std::vector<rocksdb::ColumnFamilyDescriptor, > std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, > std::vector<rocksdb::ColumnFamilyHandle*, > std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool, > bool)+0x677) [0x55716bf386cd]", > "(rocksdb::DB::Open(rocksdb::DBOptions const&, > std::__cxx11::basic_string<char, std::char_traits<char>, > std::allocator<char> > const&, > std::vector<rocksdb::ColumnFamilyDescriptor, > std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, > std::vector<rocksdb::ColumnFamilyHandle*, > std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x52) > [0x55716bf37aa4]", > "(RocksDBStore::do_open(std::ostream&, bool, bool, > std::__cxx11::basic_string<char, std::char_traits<char>, > std::allocator<char> > const&)+0x10a6) [0x55716be488b6]", > "(BlueStore::_open_db(bool, bool, bool)+0xa19) > [0x55716b8c6b19]", > "(BlueStore::_open_db_and_around(bool, bool)+0x332) > [0x55716b90bb92]", > "(BlueStore::_mount()+0x191) [0x55716b90e531]", > "(OSD::init()+0x58d) [0x55716b3b55ed]", > "main()", > "__libc_start_main()", > "_start()" > ] > } -- Jeff Layton <jlayton@xxxxxxxxxxxxxxx>
