Possibly a bug on rocksdb

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 

Dear folks,

I had an OSD down, not because of a bad disk, but most likely a bug hit on Rockdb. Any one had similar issue? 

I am using Luminous 12.2.12 version. Log attached below

thanks, 
Samuel

******************************************************************************
[root@horeb72 ceph]# head -400 ceph-osd.4.log
2019-08-11 07:30:02.186519 7f69bd020700  0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6801/4096 conn(0x56549cfc0800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.186871 7f69bd020700  0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6801/4096 conn(0x56549cfc0800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 16 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.242291 7f69bc81f700  0 -- 192.168.10.72:6805/5915 >> 192.168.10.71:6805/5046 conn(0x5654b93ed000 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.242554 7f69bc81f700  0 -- 192.168.10.72:6805/5915 >> 192.168.10.71:6805/5046 conn(0x5654b93ed000 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 16 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.260295 7f69bc81f700  0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6806/4864 conn(0x56544de16800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_CONNECTING_WAIT_CONNECT_REPLY
2019-08-11 17:11:01.968247 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
2019-08-11 17:11:01.968333 7ff4822f1d80  0 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable), process ceph-osd, pid 1048682
2019-08-11 17:11:01.970611 7ff4822f1d80  0 pidfile_write: ignore empty --pid-file
2019-08-11 17:11:01.991542 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
2019-08-11 17:11:01.997597 7ff4822f1d80  0 load: jerasure load: lrc load: isa
2019-08-11 17:11:01.997710 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
2019-08-11 17:11:01.997723 7ff4822f1d80  1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
2019-08-11 17:11:01.998127 7ff4822f1d80  1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:01.998231 7ff4822f1d80  1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) close
2019-08-11 17:11:02.265144 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
2019-08-11 17:11:02.265177 7ff4822f1d80  1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
2019-08-11 17:11:02.265695 7ff4822f1d80  1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:02.266233 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block.db type kernel
2019-08-11 17:11:02.266256 7ff4822f1d80  1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open path /var/lib/ceph/osd/ceph-4/block.db
2019-08-11 17:11:02.266812 7ff4822f1d80  1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open size 29999759360 (0x6fc200000, 27.9GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:02.266998 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
2019-08-11 17:11:02.267015 7ff4822f1d80  1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
2019-08-11 17:11:02.267412 7ff4822f1d80  1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:02.298355 7ff4822f1d80  0  set rocksdb option compaction_readahead_size = 2MB
2019-08-11 17:11:02.298368 7ff4822f1d80  0  set rocksdb option compaction_style = kCompactionStyleLevel
2019-08-11 17:11:02.299628 7ff4822f1d80  0  set rocksdb option compaction_threads = 32
2019-08-11 17:11:02.299648 7ff4822f1d80  0  set rocksdb option compression = kNoCompression
2019-08-11 17:11:02.299993 7ff4822f1d80  0  set rocksdb option flusher_threads = 8
2019-08-11 17:11:02.300006 7ff4822f1d80  0  set rocksdb option level0_file_num_compaction_trigger = 64
2019-08-11 17:11:02.300011 7ff4822f1d80  0  set rocksdb option level0_slowdown_writes_trigger = 128
2019-08-11 17:11:02.300017 7ff4822f1d80  0  set rocksdb option level0_stop_writes_trigger = 256
2019-08-11 17:11:02.300022 7ff4822f1d80  0  set rocksdb option max_background_compactions = 64
2019-08-11 17:11:02.300027 7ff4822f1d80  0  set rocksdb option max_bytes_for_level_base = 2GB
2019-08-11 17:11:02.300034 7ff4822f1d80  0  set rocksdb option max_write_buffer_number = 64
2019-08-11 17:11:02.300039 7ff4822f1d80  0  set rocksdb option min_write_buffer_number_to_merge = 32
2019-08-11 17:11:02.300044 7ff4822f1d80  0  set rocksdb option recycle_log_file_num = 64
2019-08-11 17:11:02.300048 7ff4822f1d80  0  set rocksdb option target_file_size_base = 4MB
2019-08-11 17:11:02.300053 7ff4822f1d80  0  set rocksdb option write_buffer_size = 4MB
2019-08-11 17:11:02.300093 7ff4822f1d80  0  set rocksdb option compaction_readahead_size = 2MB
2019-08-11 17:11:02.300103 7ff4822f1d80  0  set rocksdb option compaction_style = kCompactionStyleLevel
2019-08-11 17:11:02.300110 7ff4822f1d80  0  set rocksdb option compaction_threads = 32
2019-08-11 17:11:02.300121 7ff4822f1d80  0  set rocksdb option compression = kNoCompression
2019-08-11 17:11:02.300129 7ff4822f1d80  0  set rocksdb option flusher_threads = 8
2019-08-11 17:11:02.300135 7ff4822f1d80  0  set rocksdb option level0_file_num_compaction_trigger = 64
2019-08-11 17:11:02.300142 7ff4822f1d80  0  set rocksdb option level0_slowdown_writes_trigger = 128
2019-08-11 17:11:02.300146 7ff4822f1d80  0  set rocksdb option level0_stop_writes_trigger = 256
2019-08-11 17:11:02.300150 7ff4822f1d80  0  set rocksdb option max_background_compactions = 64
2019-08-11 17:11:02.300155 7ff4822f1d80  0  set rocksdb option max_bytes_for_level_base = 2GB
2019-08-11 17:11:02.300159 7ff4822f1d80  0  set rocksdb option max_write_buffer_number = 64
2019-08-11 17:11:02.300166 7ff4822f1d80  0  set rocksdb option min_write_buffer_number_to_merge = 32
2019-08-11 17:11:02.300176 7ff4822f1d80  0  set rocksdb option recycle_log_file_num = 64
2019-08-11 17:11:02.300185 7ff4822f1d80  0  set rocksdb option target_file_size_base = 4MB
2019-08-11 17:11:02.300193 7ff4822f1d80  0  set rocksdb option write_buffer_size = 4MB
2019-08-11 17:11:02.819067 7ff4822f1d80 -1 *** Caught signal (Aborted) **
 in thread 7ff4822f1d80 thread_name:ceph-osd

 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable)
 1: (()+0xa64ee1) [0x56476aafbee1]
 2: (()+0xf6d0) [0x7ff47f5a16d0]
 3: (gsignal()+0x37) [0x7ff47e5c2277]
 4: (abort()+0x148) [0x7ff47e5c3968]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7ff47eed17d5]
 6: (()+0x5e746) [0x7ff47eecf746]
 7: (()+0x5e773) [0x7ff47eecf773]
 8: (()+0x5e993) [0x7ff47eecf993]
 9: (()+0xa6f149) [0x56476ab06149]
 10: (decode(std::string&, ceph::buffer::list::iterator&)+0x53) [0x56476a765313]
 11: (OSDSuperblock::decode(ceph::buffer::list::iterator&)+0x70) [0x56476a7f23e0]
 12: (OSD::read_superblock()+0x193) [0x56476a567943]
 13: (OSD::init()+0x773) [0x56476a5b4eb3]
 14: (main()+0x2d07) [0x56476a4b7ef7]
 15: (__libc_start_main()+0xf5) [0x7ff47e5ae445]
 16: (()+0x4c0dc3) [0x56476a557dc3]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
   -68> 2019-08-11 17:11:01.963739 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perfcounters_dump hook 0x5647742e4180
   -67> 2019-08-11 17:11:01.963761 7ff4822f1d80  5 asok(0x56477431e1c0) register_command 1 hook 0x5647742e4180
   -66> 2019-08-11 17:11:01.963765 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perf dump hook 0x5647742e4180
   -65> 2019-08-11 17:11:01.963767 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perfcounters_schema hook 0x5647742e4180
   -64> 2019-08-11 17:11:01.963772 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perf histogram dump hook 0x5647742e4180
   -63> 2019-08-11 17:11:01.963775 7ff4822f1d80  5 asok(0x56477431e1c0) register_command 2 hook 0x5647742e4180
   -62> 2019-08-11 17:11:01.963777 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perf schema hook 0x5647742e4180
   -61> 2019-08-11 17:11:01.963780 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perf histogram schema hook 0x5647742e4180
   -60> 2019-08-11 17:11:01.963788 7ff4822f1d80  5 asok(0x56477431e1c0) register_command perf reset hook 0x5647742e4180
   -59> 2019-08-11 17:11:01.963792 7ff4822f1d80  5 asok(0x56477431e1c0) register_command config show hook 0x5647742e4180
   -58> 2019-08-11 17:11:01.963795 7ff4822f1d80  5 asok(0x56477431e1c0) register_command config help hook 0x5647742e4180
   -57> 2019-08-11 17:11:01.963799 7ff4822f1d80  5 asok(0x56477431e1c0) register_command config set hook 0x5647742e4180
   -56> 2019-08-11 17:11:01.963803 7ff4822f1d80  5 asok(0x56477431e1c0) register_command config get hook 0x5647742e4180
   -55> 2019-08-11 17:11:01.963806 7ff4822f1d80  5 asok(0x56477431e1c0) register_command config diff hook 0x5647742e4180
   -54> 2019-08-11 17:11:01.963809 7ff4822f1d80  5 asok(0x56477431e1c0) register_command config diff get hook 0x5647742e4180
   -53> 2019-08-11 17:11:01.963812 7ff4822f1d80  5 asok(0x56477431e1c0) register_command log flush hook 0x5647742e4180
   -52> 2019-08-11 17:11:01.963815 7ff4822f1d80  5 asok(0x56477431e1c0) register_command log dump hook 0x5647742e4180
   -51> 2019-08-11 17:11:01.963818 7ff4822f1d80  5 asok(0x56477431e1c0) register_command log reopen hook 0x5647742e4180
   -50> 2019-08-11 17:11:01.963827 7ff4822f1d80  5 asok(0x56477431e1c0) register_command dump_mempools hook 0x564774307ea8
   -49> 2019-08-11 17:11:01.968040 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
   -48> 2019-08-11 17:11:01.968247 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
   -47> 2019-08-11 17:11:01.968333 7ff4822f1d80  0 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable), process ceph-osd, pid 1048682
   -46> 2019-08-11 17:11:01.970611 7ff4822f1d80  0 pidfile_write: ignore empty --pid-file
   -45> 2019-08-11 17:11:01.991542 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
   -44> 2019-08-11 17:11:01.997597 7ff4822f1d80  0 load: jerasure load: lrc load: isa
   -43> 2019-08-11 17:11:01.997710 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
   -42> 2019-08-11 17:11:01.997723 7ff4822f1d80  1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
   -41> 2019-08-11 17:11:01.998127 7ff4822f1d80  1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
   -40> 2019-08-11 17:11:01.998231 7ff4822f1d80  1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) close
   -39> 2019-08-11 17:11:02.265144 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
   -38> 2019-08-11 17:11:02.265177 7ff4822f1d80  1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
   -37> 2019-08-11 17:11:02.265695 7ff4822f1d80  1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
   -36> 2019-08-11 17:11:02.266233 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block.db type kernel
   -35> 2019-08-11 17:11:02.266256 7ff4822f1d80  1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open path /var/lib/ceph/osd/ceph-4/block.db
   -34> 2019-08-11 17:11:02.266812 7ff4822f1d80  1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open size 29999759360 (0x6fc200000, 27.9GiB) block_size 4096 (4KiB) non-rotational
   -33> 2019-08-11 17:11:02.266998 7ff4822f1d80  1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
   -32> 2019-08-11 17:11:02.267015 7ff4822f1d80  1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
   -31> 2019-08-11 17:11:02.267412 7ff4822f1d80  1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
   -30> 2019-08-11 17:11:02.298355 7ff4822f1d80  0  set rocksdb option compaction_readahead_size = 2MB
   -29> 2019-08-11 17:11:02.298368 7ff4822f1d80  0  set rocksdb option compaction_style = kCompactionStyleLevel
   -28> 2019-08-11 17:11:02.299628 7ff4822f1d80  0  set rocksdb option compaction_threads = 32
   -27> 2019-08-11 17:11:02.299648 7ff4822f1d80  0  set rocksdb option compression = kNoCompression
   -26> 2019-08-11 17:11:02.299993 7ff4822f1d80  0  set rocksdb option flusher_threads = 8
   -25> 2019-08-11 17:11:02.300006 7ff4822f1d80  0  set rocksdb option level0_file_num_compaction_trigger = 64
   -24> 2019-08-11 17:11:02.300011 7ff4822f1d80  0  set rocksdb option level0_slowdown_writes_trigger = 128
   -23> 2019-08-11 17:11:02.300017 7ff4822f1d80  0  set rocksdb option level0_stop_writes_trigger = 256
   -22> 2019-08-11 17:11:02.300022 7ff4822f1d80  0  set rocksdb option max_background_compactions = 64
   -21> 2019-08-11 17:11:02.300027 7ff4822f1d80  0  set rocksdb option max_bytes_for_level_base = 2GB
   -20> 2019-08-11 17:11:02.300034 7ff4822f1d80  0  set rocksdb option max_write_buffer_number = 64
   -19> 2019-08-11 17:11:02.300039 7ff4822f1d80  0  set rocksdb option min_write_buffer_number_to_merge = 32
   -18> 2019-08-11 17:11:02.300044 7ff4822f1d80  0  set rocksdb option recycle_log_file_num = 64
   -17> 2019-08-11 17:11:02.300048 7ff4822f1d80  0  set rocksdb option target_file_size_base = 4MB
   -16> 2019-08-11 17:11:02.300053 7ff4822f1d80  0  set rocksdb option write_buffer_size = 4MB
   -15> 2019-08-11 17:11:02.300093 7ff4822f1d80  0  set rocksdb option compaction_readahead_size = 2MB
   -14> 2019-08-11 17:11:02.300103 7ff4822f1d80  0  set rocksdb option compaction_style = kCompactionStyleLevel
   -13> 2019-08-11 17:11:02.300110 7ff4822f1d80  0  set rocksdb option compaction_threads = 32
   -12> 2019-08-11 17:11:02.300121 7ff4822f1d80  0  set rocksdb option compression = kNoCompression
   -11> 2019-08-11 17:11:02.300129 7ff4822f1d80  0  set rocksdb option flusher_threads = 8
   -10> 2019-08-11 17:11:02.300135 7ff4822f1d80  0  set rocksdb option level0_file_num_compaction_trigger = 64
    -9> 2019-08-11 17:11:02.300142 7ff4822f1d80  0  set rocksdb option level0_slowdown_writes_trigger = 128
    -8> 2019-08-11 17:11:02.300146 7ff4822f1d80  0  set rocksdb option level0_stop_writes_trigger = 256
    -7> 2019-08-11 17:11:02.300150 7ff4822f1d80  0  set rocksdb option max_background_compactions = 64
    -6> 2019-08-11 17:11:02.300155 7ff4822f1d80  0  set rocksdb option max_bytes_for_level_base = 2GB
    -5> 2019-08-11 17:11:02.300159 7ff4822f1d80  0  set rocksdb option max_write_buffer_number = 64
    -4> 2019-08-11 17:11:02.300166 7ff4822f1d80  0  set rocksdb option min_write_buffer_number_to_merge = 32
    -3> 2019-08-11 17:11:02.300176 7ff4822f1d80  0  set rocksdb option recycle_log_file_num = 64
    -2> 2019-08-11 17:11:02.300185 7ff4822f1d80  0  set rocksdb option target_file_size_base = 4MB
    -1> 2019-08-11 17:11:02.300193 7ff4822f1d80  0  set rocksdb option write_buffer_size = 4MB
     0> 2019-08-11 17:11:02.819067 7ff4822f1d80 -1 *** Caught signal (Aborted) **
 in thread 7ff4822f1d80 thread_name:ceph-osd
huxiaoyu@xxxxxxxxxxxx
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
[Index of Archives]     [Information on CEPH]     [Linux Filesystem Development]     [Ceph Development]     [Ceph Large]     [Ceph Dev]     [Linux USB Development]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]     [xfs]

  Powered by Linux