Hi Folks,
I'm having trouble getting some of my OSDs to boot. At some point, these
disks got very full. I fixed the rule that was causing that, and they
are on average ~30% full now.
I'm getting the following in my logs:
-1> 2019-05-08 16:05:18.956 7fdc7adbbf00 -1
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.1/rpm/el7/BUILD/ceph-14.2.1/src/include/interval_set.h:
In function 'void interval_set<T, Map>::insert(T, T, T*, T*) [with T =
long unsigned int; Map = std::map<long unsigned int, long unsigned int,
std::less<long unsigned int>, std::allocator<std::pair<const long
unsigned int, long unsigned int> > >]' thread 7fdc7adbbf00 time
2019-05-08 16:05:18.953372
/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.1/rpm/el7/BUILD/ceph-14.2.1/src/include/interval_set.h:
490: FAILED ceph_assert(p->first > start+len)
ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972)
nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x14a) [0x7fdc70daa676]
2: (ceph::__ceph_assertf_fail(char const*, char const*, int, char
const*, char const*, ...)+0) [0x7fdc70daa844]
3: (interval_set<unsigned long, std::map<unsigned long, unsigned long,
std::less<unsigned long>, std::allocator<std::pair<unsigned long const,
unsigned long> > > >::insert(unsigned long, unsigned long, unsigned
long*, unsigned long*)+0x45f) [0x55b8960e03df]
4: (BlueStore::allocate_bluefs_freespace(unsigned long, unsigned long,
std::vector<bluestore_pextent_t,
mempool::pool_allocator<(mempool::pool_index_t)4, bluestore_pextent_t>
>*)+0x74e) [0x55b89611d13e]
5: (BlueFS::_expand_slow_device(unsigned long,
std::vector<bluestore_pextent_t,
mempool::pool_allocator<(mempool::pool_index_t)4, bluestore_pextent_t>
>&)+0x111) [0x55b8960c8211]
6: (BlueFS::_allocate(unsigned char, unsigned long,
bluefs_fnode_t*)+0x68b) [0x55b8960c8f7b]
7: (BlueFS::_allocate(unsigned char, unsigned long,
bluefs_fnode_t*)+0x362) [0x55b8960c8c52]
8: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned
long)+0xe5) [0x55b8960c95d5]
9: (BlueFS::_flush(BlueFS::FileWriter*, bool)+0x10b) [0x55b8960cb43b]
10: (BlueRocksWritableFile::Flush()+0x3d) [0x55b8962bdfcd]
11: (rocksdb::WritableFileWriter::Flush()+0x19e) [0x55b896531a4e]
12: (rocksdb::WritableFileWriter::Sync(bool)+0x2e) [0x55b896531d2e]
13: (rocksdb::BuildTable(std::string const&, rocksdb::Env*,
rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&,
rocksdb::EnvOptions const&, rocksdb::TableCache*,
rocksdb::InternalIteratorBase<rocksdb::Slice>*,
std::unique_ptr<rocksdb::InternalIteratorBase<rocksdb::Slice>,
std::default_delete<rocksdb::InternalIteratorBase<rocksdb::Slice> > >,
rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&,
std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory,
std::default_delete<rocksdb::IntTblPropCollectorFactory> >,
std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory,
std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*,
unsigned int, std::string const&, std::vector<unsigned long,
std::allocator<unsigned long> >, unsigned long,
rocksdb::SnapshotChecker*, rocksdb::CompressionType,
rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*,
rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int,
rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long,
unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2368) [0x55b89655fb68]
14: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int,
rocksdb::ColumnFamilyData*, rocksdb::MemTable*,
rocksdb::VersionEdit*)+0xc66) [0x55b8963d48c6]
15: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long,
std::allocator<unsigned long> > const&, unsigned long*, bool)+0x1dce)
[0x55b8963d6f1e]
16:
(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor,
std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool,
bool)+0x809) [0x55b8963d7db9]
17: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::string
const&, std::vector<rocksdb::ColumnFamilyDescriptor,
std::allocator<rocksdb::ColumnFamilyDescriptor> > const&,
std::vector<rocksdb::ColumnFamilyHandle*,
std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool,
bool)+0x658) [0x55b8963d8bc8]
18: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::string const&,
std::vector<rocksdb::ColumnFamilyDescriptor,
std::allocator<rocksdb::ColumnFamilyDescriptor> > const&,
std::vector<rocksdb::ColumnFamilyHandle*,
std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x24)
[0x55b8963da3a4]
19: (RocksDBStore::do_open(std::ostream&, bool, bool,
std::vector<KeyValueDB::ColumnFamily,
std::allocator<KeyValueDB::ColumnFamily> > const*)+0x1660) [0x55b8961c2a80]
20: (BlueStore::_open_db(bool, bool, bool)+0xf8e) [0x55b89611b37e]
21: (BlueStore::_open_db_and_around(bool)+0x165) [0x55b8961388b5]
22: (BlueStore::_fsck(bool, bool)+0xe5c) [0x55b8961692dc]
23: (main()+0x107e) [0x55b895fc682e]
24: (__libc_start_main()+0xf5) [0x7fdc6da4e3d5]
25: (()+0x2718cf) [0x55b8960ac8cf]
0> 2019-05-08 16:05:18.960 7fdc7adbbf00 -1 *** Caught signal
(Aborted) **
in thread 7fdc7adbbf00 thread_name:ceph-bluestore-
ceph version 14.2.1 (d555a9489eb35f84f2e1ef49b77e19da9d113972)
nautilus (stable)
1: (()+0xf5d0) [0x7fdc6f2905d0]
2: (gsignal()+0x37) [0x7fdc6da62207]
3: (abort()+0x148) [0x7fdc6da638f8]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x199) [0x7fdc70daa6c5]
5: (ceph::__ceph_assertf_fail(char const*, char const*, int, char
const*, char const*, ...)+0) [0x7fdc70daa844]
6: (interval_set<unsigned long, std::map<unsigned long, unsigned long,
std::less<unsigned long>, std::allocator<std::pair<unsigned long const,
unsigned long> > > >::insert(unsigned long, unsigned long, unsigned
long*, unsigned long*)+0x45f) [0x55b8960e03df]
7: (BlueStore::allocate_bluefs_freespace(unsigned long, unsigned long,
std::vector<bluestore_pextent_t,
mempool::pool_allocator<(mempool::pool_index_t)4, bluestore_pextent_t>
>*)+0x74e) [0x55b89611d13e]
8: (BlueFS::_expand_slow_device(unsigned long,
std::vector<bluestore_pextent_t,
mempool::pool_allocator<(mempool::pool_index_t)4, bluestore_pextent_t>
>&)+0x111) [0x55b8960c8211]
9: (BlueFS::_allocate(unsigned char, unsigned long,
bluefs_fnode_t*)+0x68b) [0x55b8960c8f7b]
10: (BlueFS::_allocate(unsigned char, unsigned long,
bluefs_fnode_t*)+0x362) [0x55b8960c8c52]
11: (BlueFS::_flush_range(BlueFS::FileWriter*, unsigned long, unsigned
long)+0xe5) [0x55b8960c95d5]
12: (BlueFS::_flush(BlueFS::FileWriter*, bool)+0x10b) [0x55b8960cb43b]
13: (BlueRocksWritableFile::Flush()+0x3d) [0x55b8962bdfcd]
14: (rocksdb::WritableFileWriter::Flush()+0x19e) [0x55b896531a4e]
15: (rocksdb::WritableFileWriter::Sync(bool)+0x2e) [0x55b896531d2e]
16: (rocksdb::BuildTable(std::string const&, rocksdb::Env*,
rocksdb::ImmutableCFOptions const&, rocksdb::MutableCFOptions const&,
rocksdb::EnvOptions const&, rocksdb::TableCache*,
rocksdb::InternalIteratorBase<rocksdb::Slice>*,
std::unique_ptr<rocksdb::InternalIteratorBase<rocksdb::Slice>,
std::default_delete<rocksdb::InternalIteratorBase<rocksdb::Slice> > >,
rocksdb::FileMetaData*, rocksdb::InternalKeyComparator const&,
std::vector<std::unique_ptr<rocksdb::IntTblPropCollectorFactory,
std::default_delete<rocksdb::IntTblPropCollectorFactory> >,
std::allocator<std::unique_ptr<rocksdb::IntTblPropCollectorFactory,
std::default_delete<rocksdb::IntTblPropCollectorFactory> > > > const*,
unsigned int, std::string const&, std::vector<unsigned long,
std::allocator<unsigned long> >, unsigned long,
rocksdb::SnapshotChecker*, rocksdb::CompressionType,
rocksdb::CompressionOptions const&, bool, rocksdb::InternalStats*,
rocksdb::TableFileCreationReason, rocksdb::EventLogger*, int,
rocksdb::Env::IOPriority, rocksdb::TableProperties*, int, unsigned long,
unsigned long, rocksdb::Env::WriteLifeTimeHint)+0x2368) [0x55b89655fb68]
17: (rocksdb::DBImpl::WriteLevel0TableForRecovery(int,
rocksdb::ColumnFamilyData*, rocksdb::MemTable*,
rocksdb::VersionEdit*)+0xc66) [0x55b8963d48c6]
18: (rocksdb::DBImpl::RecoverLogFiles(std::vector<unsigned long,
std::allocator<unsigned long> > const&, unsigned long*, bool)+0x1dce)
[0x55b8963d6f1e]
19:
(rocksdb::DBImpl::Recover(std::vector<rocksdb::ColumnFamilyDescriptor,
std::allocator<rocksdb::ColumnFamilyDescriptor> > const&, bool, bool,
bool)+0x809) [0x55b8963d7db9]
20: (rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::string
const&, std::vector<rocksdb::ColumnFamilyDescriptor,
std::allocator<rocksdb::ColumnFamilyDescriptor> > const&,
std::vector<rocksdb::ColumnFamilyHandle*,
std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**, bool,
bool)+0x658) [0x55b8963d8bc8]
21: (rocksdb::DB::Open(rocksdb::DBOptions const&, std::string const&,
std::vector<rocksdb::ColumnFamilyDescriptor,
std::allocator<rocksdb::ColumnFamilyDescriptor> > const&,
std::vector<rocksdb::ColumnFamilyHandle*,
std::allocator<rocksdb::ColumnFamilyHandle*> >*, rocksdb::DB**)+0x24)
[0x55b8963da3a4]
22: (RocksDBStore::do_open(std::ostream&, bool, bool,
std::vector<KeyValueDB::ColumnFamily,
std::allocator<KeyValueDB::ColumnFamily> > const*)+0x1660) [0x55b8961c2a80]
23: (BlueStore::_open_db(bool, bool, bool)+0xf8e) [0x55b89611b37e]
24: (BlueStore::_open_db_and_around(bool)+0x165) [0x55b8961388b5]
25: (BlueStore::_fsck(bool, bool)+0xe5c) [0x55b8961692dc]
26: (main()+0x107e) [0x55b895fc682e]
27: (__libc_start_main()+0xf5) [0x7fdc6da4e3d5]
28: (()+0x2718cf) [0x55b8960ac8cf]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is
needed to interpret this.
The osds were provisioned using
ceph-volume prepare --dmcrypt --data /dev/sdx
It looks to me like one of the db/wal/main claimed all space while the
osd was full and hasn't released it -- leaving none left for the others.
I've tried extracting pgs with ceph-objectstore tool, but get the same
stack trace.
Any help would be greatly appreciated!
-Paul
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
