Could you create a tracker for this? Also, if you can reproduce this could you gather a log with debug_osd=20 ? That should show us the superblock it was trying to decode as well as additional details. On Mon, Aug 12, 2019 at 6:29 AM huxiaoyu@xxxxxxxxxxxx <huxiaoyu@xxxxxxxxxxxx> wrote: > > Dear folks, > > I had an OSD down, not because of a bad disk, but most likely a bug hit on Rockdb. Any one had similar issue? > > I am using Luminous 12.2.12 version. Log attached below > > thanks, > Samuel > > ****************************************************************************** > [root@horeb72 ceph]# head -400 ceph-osd.4.log > 2019-08-11 07:30:02.186519 7f69bd020700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6801/4096 conn(0x56549cfc0800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_STANDBY > 2019-08-11 07:30:02.186871 7f69bd020700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6801/4096 conn(0x56549cfc0800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 16 vs existing csq=15 existing_state=STATE_STANDBY > 2019-08-11 07:30:02.242291 7f69bc81f700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.71:6805/5046 conn(0x5654b93ed000 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_STANDBY > 2019-08-11 07:30:02.242554 7f69bc81f700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.71:6805/5046 conn(0x5654b93ed000 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 16 vs existing csq=15 existing_state=STATE_STANDBY > 2019-08-11 07:30:02.260295 7f69bc81f700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6806/4864 conn(0x56544de16800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_CONNECTING_WAIT_CONNECT_REPLY > 2019-08-11 17:11:01.968247 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb > 2019-08-11 17:11:01.968333 7ff4822f1d80 0 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable), process ceph-osd, pid 1048682 > 2019-08-11 17:11:01.970611 7ff4822f1d80 0 pidfile_write: ignore empty --pid-file > 2019-08-11 17:11:01.991542 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb > 2019-08-11 17:11:01.997597 7ff4822f1d80 0 load: jerasure load: lrc load: isa > 2019-08-11 17:11:01.997710 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel > 2019-08-11 17:11:01.997723 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block > 2019-08-11 17:11:01.998127 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational > 2019-08-11 17:11:01.998231 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) close > 2019-08-11 17:11:02.265144 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel > 2019-08-11 17:11:02.265177 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block > 2019-08-11 17:11:02.265695 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational > 2019-08-11 17:11:02.266233 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block.db type kernel > 2019-08-11 17:11:02.266256 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open path /var/lib/ceph/osd/ceph-4/block.db > 2019-08-11 17:11:02.266812 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open size 29999759360 (0x6fc200000, 27.9GiB) block_size 4096 (4KiB) non-rotational > 2019-08-11 17:11:02.266998 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel > 2019-08-11 17:11:02.267015 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block > 2019-08-11 17:11:02.267412 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational > 2019-08-11 17:11:02.298355 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB > 2019-08-11 17:11:02.298368 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel > 2019-08-11 17:11:02.299628 7ff4822f1d80 0 set rocksdb option compaction_threads = 32 > 2019-08-11 17:11:02.299648 7ff4822f1d80 0 set rocksdb option compression = kNoCompression > 2019-08-11 17:11:02.299993 7ff4822f1d80 0 set rocksdb option flusher_threads = 8 > 2019-08-11 17:11:02.300006 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64 > 2019-08-11 17:11:02.300011 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128 > 2019-08-11 17:11:02.300017 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256 > 2019-08-11 17:11:02.300022 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64 > 2019-08-11 17:11:02.300027 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB > 2019-08-11 17:11:02.300034 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64 > 2019-08-11 17:11:02.300039 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32 > 2019-08-11 17:11:02.300044 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64 > 2019-08-11 17:11:02.300048 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB > 2019-08-11 17:11:02.300053 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB > 2019-08-11 17:11:02.300093 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB > 2019-08-11 17:11:02.300103 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel > 2019-08-11 17:11:02.300110 7ff4822f1d80 0 set rocksdb option compaction_threads = 32 > 2019-08-11 17:11:02.300121 7ff4822f1d80 0 set rocksdb option compression = kNoCompression > 2019-08-11 17:11:02.300129 7ff4822f1d80 0 set rocksdb option flusher_threads = 8 > 2019-08-11 17:11:02.300135 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64 > 2019-08-11 17:11:02.300142 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128 > 2019-08-11 17:11:02.300146 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256 > 2019-08-11 17:11:02.300150 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64 > 2019-08-11 17:11:02.300155 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB > 2019-08-11 17:11:02.300159 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64 > 2019-08-11 17:11:02.300166 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32 > 2019-08-11 17:11:02.300176 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64 > 2019-08-11 17:11:02.300185 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB > 2019-08-11 17:11:02.300193 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB > 2019-08-11 17:11:02.819067 7ff4822f1d80 -1 *** Caught signal (Aborted) ** > in thread 7ff4822f1d80 thread_name:ceph-osd > > ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable) > 1: (()+0xa64ee1) [0x56476aafbee1] > 2: (()+0xf6d0) [0x7ff47f5a16d0] > 3: (gsignal()+0x37) [0x7ff47e5c2277] > 4: (abort()+0x148) [0x7ff47e5c3968] > 5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7ff47eed17d5] > 6: (()+0x5e746) [0x7ff47eecf746] > 7: (()+0x5e773) [0x7ff47eecf773] > 8: (()+0x5e993) [0x7ff47eecf993] > 9: (()+0xa6f149) [0x56476ab06149] > 10: (decode(std::string&, ceph::buffer::list::iterator&)+0x53) [0x56476a765313] > 11: (OSDSuperblock::decode(ceph::buffer::list::iterator&)+0x70) [0x56476a7f23e0] > 12: (OSD::read_superblock()+0x193) [0x56476a567943] > 13: (OSD::init()+0x773) [0x56476a5b4eb3] > 14: (main()+0x2d07) [0x56476a4b7ef7] > 15: (__libc_start_main()+0xf5) [0x7ff47e5ae445] > 16: (()+0x4c0dc3) [0x56476a557dc3] > NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. > > --- begin dump of recent events --- > -68> 2019-08-11 17:11:01.963739 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perfcounters_dump hook 0x5647742e4180 > -67> 2019-08-11 17:11:01.963761 7ff4822f1d80 5 asok(0x56477431e1c0) register_command 1 hook 0x5647742e4180 > -66> 2019-08-11 17:11:01.963765 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf dump hook 0x5647742e4180 > -65> 2019-08-11 17:11:01.963767 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perfcounters_schema hook 0x5647742e4180 > -64> 2019-08-11 17:11:01.963772 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf histogram dump hook 0x5647742e4180 > -63> 2019-08-11 17:11:01.963775 7ff4822f1d80 5 asok(0x56477431e1c0) register_command 2 hook 0x5647742e4180 > -62> 2019-08-11 17:11:01.963777 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf schema hook 0x5647742e4180 > -61> 2019-08-11 17:11:01.963780 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf histogram schema hook 0x5647742e4180 > -60> 2019-08-11 17:11:01.963788 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf reset hook 0x5647742e4180 > -59> 2019-08-11 17:11:01.963792 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config show hook 0x5647742e4180 > -58> 2019-08-11 17:11:01.963795 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config help hook 0x5647742e4180 > -57> 2019-08-11 17:11:01.963799 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config set hook 0x5647742e4180 > -56> 2019-08-11 17:11:01.963803 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config get hook 0x5647742e4180 > -55> 2019-08-11 17:11:01.963806 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config diff hook 0x5647742e4180 > -54> 2019-08-11 17:11:01.963809 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config diff get hook 0x5647742e4180 > -53> 2019-08-11 17:11:01.963812 7ff4822f1d80 5 asok(0x56477431e1c0) register_command log flush hook 0x5647742e4180 > -52> 2019-08-11 17:11:01.963815 7ff4822f1d80 5 asok(0x56477431e1c0) register_command log dump hook 0x5647742e4180 > -51> 2019-08-11 17:11:01.963818 7ff4822f1d80 5 asok(0x56477431e1c0) register_command log reopen hook 0x5647742e4180 > -50> 2019-08-11 17:11:01.963827 7ff4822f1d80 5 asok(0x56477431e1c0) register_command dump_mempools hook 0x564774307ea8 > -49> 2019-08-11 17:11:01.968040 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb > -48> 2019-08-11 17:11:01.968247 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb > -47> 2019-08-11 17:11:01.968333 7ff4822f1d80 0 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable), process ceph-osd, pid 1048682 > -46> 2019-08-11 17:11:01.970611 7ff4822f1d80 0 pidfile_write: ignore empty --pid-file > -45> 2019-08-11 17:11:01.991542 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb > -44> 2019-08-11 17:11:01.997597 7ff4822f1d80 0 load: jerasure load: lrc load: isa > -43> 2019-08-11 17:11:01.997710 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel > -42> 2019-08-11 17:11:01.997723 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block > -41> 2019-08-11 17:11:01.998127 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational > -40> 2019-08-11 17:11:01.998231 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) close > -39> 2019-08-11 17:11:02.265144 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel > -38> 2019-08-11 17:11:02.265177 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block > -37> 2019-08-11 17:11:02.265695 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational > -36> 2019-08-11 17:11:02.266233 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block.db type kernel > -35> 2019-08-11 17:11:02.266256 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open path /var/lib/ceph/osd/ceph-4/block.db > -34> 2019-08-11 17:11:02.266812 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open size 29999759360 (0x6fc200000, 27.9GiB) block_size 4096 (4KiB) non-rotational > -33> 2019-08-11 17:11:02.266998 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel > -32> 2019-08-11 17:11:02.267015 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block > -31> 2019-08-11 17:11:02.267412 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational > -30> 2019-08-11 17:11:02.298355 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB > -29> 2019-08-11 17:11:02.298368 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel > -28> 2019-08-11 17:11:02.299628 7ff4822f1d80 0 set rocksdb option compaction_threads = 32 > -27> 2019-08-11 17:11:02.299648 7ff4822f1d80 0 set rocksdb option compression = kNoCompression > -26> 2019-08-11 17:11:02.299993 7ff4822f1d80 0 set rocksdb option flusher_threads = 8 > -25> 2019-08-11 17:11:02.300006 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64 > -24> 2019-08-11 17:11:02.300011 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128 > -23> 2019-08-11 17:11:02.300017 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256 > -22> 2019-08-11 17:11:02.300022 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64 > -21> 2019-08-11 17:11:02.300027 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB > -20> 2019-08-11 17:11:02.300034 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64 > -19> 2019-08-11 17:11:02.300039 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32 > -18> 2019-08-11 17:11:02.300044 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64 > -17> 2019-08-11 17:11:02.300048 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB > -16> 2019-08-11 17:11:02.300053 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB > -15> 2019-08-11 17:11:02.300093 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB > -14> 2019-08-11 17:11:02.300103 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel > -13> 2019-08-11 17:11:02.300110 7ff4822f1d80 0 set rocksdb option compaction_threads = 32 > -12> 2019-08-11 17:11:02.300121 7ff4822f1d80 0 set rocksdb option compression = kNoCompression > -11> 2019-08-11 17:11:02.300129 7ff4822f1d80 0 set rocksdb option flusher_threads = 8 > -10> 2019-08-11 17:11:02.300135 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64 > -9> 2019-08-11 17:11:02.300142 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128 > -8> 2019-08-11 17:11:02.300146 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256 > -7> 2019-08-11 17:11:02.300150 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64 > -6> 2019-08-11 17:11:02.300155 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB > -5> 2019-08-11 17:11:02.300159 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64 > -4> 2019-08-11 17:11:02.300166 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32 > -3> 2019-08-11 17:11:02.300176 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64 > -2> 2019-08-11 17:11:02.300185 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB > -1> 2019-08-11 17:11:02.300193 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB > 0> 2019-08-11 17:11:02.819067 7ff4822f1d80 -1 *** Caught signal (Aborted) ** > in thread 7ff4822f1d80 thread_name:ceph-osd > ________________________________ > huxiaoyu@xxxxxxxxxxxx > _______________________________________________ > ceph-users mailing list > ceph-users@xxxxxxxxxxxxxx > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com -- Cheers, Brad _______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
 |