|
Hello,
We have a cluster in version 17.2.5 and on 2 nodes (out of 5), some metrics are at 0 on all OSD. The problem is really global to the node.
I performed an upgrade to 17.2.6 on the cluster like this :
This fixed for one node of the cluster but not the other so I stil don't understand.
Here is an example of such OSD :
{
"AsyncMessenger::Worker-0": {
"msgr_recv_messages": 6843132,
"msgr_send_messages": 6880681,
"msgr_recv_bytes": 63926805259,
"msgr_send_bytes": 3298578019,
"msgr_created_connections": 354,
"msgr_active_connections": 20,
"msgr_running_total_time": 361.495252541,
"msgr_running_send_time": 88.204996929,
"msgr_running_recv_time": 157.610653554,
"msgr_running_fast_dispatch_time": 94.944377551,
"msgr_send_messages_queue_lat": {
"avgcount": 6880679,
"sum": 398.607948692,
"avgtime": 0.000057931
},
"msgr_handle_ack_lat": {
"avgcount": 8509969,
"sum": 1.558485881,
"avgtime": 0.000000183
}
},
"AsyncMessenger::Worker-1": {
"msgr_recv_messages": 2332821,
"msgr_send_messages": 2357957,
"msgr_recv_bytes": 17554824362,
"msgr_send_bytes": 3407896999,
"msgr_created_connections": 177,
"msgr_active_connections": 18,
"msgr_running_total_time": 112.862256919,
"msgr_running_send_time": 34.874196263,
"msgr_running_recv_time": 46.995625507,
"msgr_running_fast_dispatch_time": 25.082966322,
"msgr_send_messages_queue_lat": {
"avgcount": 2357955,
"sum": 85.725096545,
"avgtime": 0.000036355
},
"msgr_handle_ack_lat": {
"avgcount": 1140003,
"sum": 0.241074745,
"avgtime": 0.000000211
}
},
"AsyncMessenger::Worker-2": {
"msgr_recv_messages": 4326174,
"msgr_send_messages": 4330697,
"msgr_recv_bytes": 31752213275,
"msgr_send_bytes": 3567666813,
"msgr_created_connections": 100,
"msgr_active_connections": 20,
"msgr_running_total_time": 218.260025879,
"msgr_running_send_time": 58.635931148,
"msgr_running_recv_time": 92.152198211,
"msgr_running_fast_dispatch_time": 54.438590557,
"msgr_send_messages_queue_lat": {
"avgcount": 4330696,
"sum": 250.493265066,
"avgtime": 0.000057841
},
"msgr_handle_ack_lat": {
"avgcount": 4229647,
"sum": 0.806930547,
"avgtime": 0.000000190
}
},
"bluefs": {
"db_total_bytes": 1920362086400,
"db_used_bytes": 1089011712,
"wal_total_bytes": 0,
"wal_used_bytes": 0,
"slow_total_bytes": 0,
"slow_used_bytes": 0,
"num_files": 125,
"log_bytes": 2646016,
"log_compactions": 812,
"logged_bytes": 13500940288,
"files_written_wal": 1575,
"files_written_sst": 182,
"bytes_written_wal": 25542598656,
"bytes_written_sst": 2466263040,
"bytes_written_slow": 0,
"max_bytes_wal": 0,
"max_bytes_db": 2035351552,
"max_bytes_slow": 0,
"alloc_unit_main": 0,
"alloc_unit_db": 65536,
"alloc_unit_wal": 0,
"read_random_count": 201339,
"read_random_bytes": 3080889493,
"read_random_disk_count": 101652,
"read_random_disk_bytes": 2645938201,
"read_random_disk_bytes_wal": 0,
"read_random_disk_bytes_db": 2645938201,
"read_random_disk_bytes_slow": 0,
"read_random_buffer_count": 110722,
"read_random_buffer_bytes": 434951292,
"read_count": 80250,
"read_bytes": 1159472505,
"read_disk_count": 68083,
"read_disk_bytes": 1053351936,
"read_disk_bytes_wal": 0,
"read_disk_bytes_db": 1053356032,
"read_disk_bytes_slow": 0,
"read_prefetch_count": 78665,
"read_prefetch_bytes": 1108068939,
"compact_lat": {
"avgcount": 812,
"sum": 1.246095373,
"avgtime": 0.001534600
},
"compact_lock_lat": {
"avgcount": 812,
"sum": 0.466894228,
"avgtime": 0.000574992
},
"alloc_slow_fallback": 0,
"alloc_slow_size_fallback": 0,
"read_zeros_candidate": 0,
"read_zeros_errors": 0
},
"bluestore": {
"allocated": 180780802048,
"stored": 188948817556,
"fragmentation_micros": 2,
"alloc_unit": 4096,
"state_prepare_lat": {
"avgcount": 3979669,
"sum": 426.352439741,
"avgtime": 0.000107132
},
"state_aio_wait_lat": {
"avgcount": 3979669,
"sum": 662.940570063,
"avgtime": 0.000166581
},
"state_io_done_lat": {
"avgcount": 3979669,
"sum": 7.323425924,
"avgtime": 0.000001840
},
"state_kv_queued_lat": {
"avgcount": 3979669,
"sum": 1900.856846444,
"avgtime": 0.000477641
},
"state_kv_commiting_lat": {
"avgcount": 3979669,
"sum": 3192.442512062,
"avgtime": 0.000802187
},
"state_kv_done_lat": {
"avgcount": 3979669,
"sum": 1.143240407,
"avgtime": 0.000000287
},
"state_finishing_lat": {
"avgcount": 3979665,
"sum": 0.363208449,
"avgtime": 0.000000091
},
"state_done_lat": {
"avgcount": 3979658,
"sum": 123255.522846135,
"avgtime": 0.030971385
},
"state_deferred_queued_lat": {
"avgcount": 356134,
"sum": 455582.851504787,
"avgtime": 1.279245597
},
"state_deferred_aio_wait_lat": {
"avgcount": 356134,
"sum": 64.258012775,
"avgtime": 0.000180432
},
"state_deferred_cleanup_lat": {
"avgcount": 356134,
"sum": 58342.990966896,
"avgtime": 0.163823142
},
"txc_commit_lat": {
"avgcount": 3979669,
"sum": 6190.519490871,
"avgtime": 0.001555536
},
"txc_throttle_lat": {
"avgcount": 3979669,
"sum": 61.112847717,
"avgtime": 0.000015356
},
"txc_submit_lat": {
"avgcount": 3979669,
"sum": 495.416650342,
"avgtime": 0.000124486
},
"txc_count": 3979669,
"read_onode_meta_lat": {
"avgcount": 1197781,
"sum": 19.722806609,
"avgtime": 0.000016466
},
"read_wait_aio_lat": {
"avgcount": 929167,
"sum": 537.061469380,
"avgtime": 0.000578003
},
"csum_lat": {
"avgcount": 2133026,
"sum": 13.996152575,
"avgtime": 0.000006561
},
"read_eio": 0,
"reads_with_retries": 0,
"read_lat": {
"avgcount": 268614,
"sum": 520.246577964,
"avgtime": 0.001936781
},
"kv_flush_lat": {
"avgcount": 3291697,
"sum": 403.833685777,
"avgtime": 0.000122682
},
"kv_commit_lat": {
"avgcount": 3291697,
"sum": 2311.502280053,
"avgtime": 0.000702222
},
"kv_sync_lat": {
"avgcount": 3291697,
"sum": 2715.335965830,
"avgtime": 0.000824904
},
"kv_final_lat": {
"avgcount": 3291667,
"sum": 65.461458609,
"avgtime": 0.000019887
},
"write_big": 3712142,
"write_big_bytes": 94437191680,
"write_big_blobs": 4685989,
"write_big_deferred": 0,
"write_small": 584115,
"write_small_bytes": 960510935,
"write_small_unused": 13820,
"write_small_pre_read": 567335,
"write_pad_bytes": 217785305,
"write_penalty_read_ops": 688111,
"write_new": 4702769,
"issued_deferred_writes": 567335,
"issued_deferred_write_bytes": 2323804160,
"submitted_deferred_writes": 251455,
"submitted_deferred_write_bytes": 1235841024,
"write_big_skipped_blobs": 0,
"write_big_skipped_bytes": 0,
"write_small_skipped": 0,
"write_small_skipped_bytes": 0,
"compressed": 0,
"compressed_allocated": 0,
"compressed_original": 0,
"compress_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"decompress_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"compress_success_count": 0,
"compress_rejected_count": 0,
"onodes": 42417,
"onodes_pinned": 0,
"onode_hits": 8904311,
"onode_misses": 87317,
"onode_shard_hits": 6533806,
"onode_shard_misses": 270931,
"onode_extents": 1288567,
"onode_blobs": 1116911,
"buffers": 956,
"buffer_bytes": 4562944,
"buffer_hit_bytes": 652455498,
"buffer_miss_bytes": 115020768720,
"onode_reshard": 54721,
"blob_split": 1236,
"extent_compress": 4199105,
"gc_merged": 0,
"omap_seek_to_first_lat": {
"avgcount": 31054,
"sum": 1.622677452,
"avgtime": 0.000052253
},
"omap_upper_bound_lat": {
"avgcount": 4,
"sum": 0.016788155,
"avgtime": 0.004197038
},
"omap_lower_bound_lat": {
"avgcount": 82,
"sum": 0.001473844,
"avgtime": 0.000017973
},
"omap_next_lat": {
"avgcount": 791854,
"sum": 2.400923607,
"avgtime": 0.000003032
},
"omap_get_keys_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"omap_get_values_lat": {
"avgcount": 26905,
"sum": 4.060382074,
"avgtime": 0.000150915
},
"omap_clear_lat": {
"avgcount": 3,
"sum": 0.001249818,
"avgtime": 0.000416606
},
"clist_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"remove_lat": {
"avgcount": 35724,
"sum": 28.130138084,
"avgtime": 0.000787429
},
"truncate_lat": {
"avgcount": 6300,
"sum": 0.361881579,
"avgtime": 0.000057441
}
},
"bluestore-pricache": {
"target_bytes": 4294967296,
"mapped_bytes": 2529673216,
"unmapped_bytes": 230645760,
"heap_bytes": 2760318976,
"cache_bytes": 2845415832
},
"bluestore-pricache:data": {
"pri0_bytes": 0,
"pri1_bytes": 0,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 24576,
"pri5_bytes": 57344,
"pri6_bytes": 81920,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 77126679,
"reserved_bytes": 73704425,
"committed_bytes": 150994944
},
"bluestore-pricache:kv": {
"pri0_bytes": 67108864,
"pri1_bytes": 0,
"pri2_bytes": 0,
"pri3_bytes": 4375736,
"pri4_bytes": 0,
"pri5_bytes": 112032,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 578450095,
"reserved_bytes": 71373561,
"committed_bytes": 721420288
},
"bluestore-pricache:kv_onode": {
"pri0_bytes": 67108864,
"pri1_bytes": 0,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 5099312,
"pri11_bytes": 51417786,
"reserved_bytes": 77700630,
"committed_bytes": 201326592
},
"bluestore-pricache:meta": {
"pri0_bytes": 0,
"pri1_bytes": 90067,
"pri2_bytes": 90067,
"pri3_bytes": 1200904,
"pri4_bytes": 1486119,
"pri5_bytes": 3047296,
"pri6_bytes": 3632737,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 578450095,
"reserved_bytes": 83091355,
"committed_bytes": 671088640
},
"cct": {
"total_workers": 16,
"unhealthy_workers": 0
},
"finisher-commit_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 286,
"sum": 0.262805597,
"avgtime": 0.000918900
}
},
"finisher-objecter-finisher-0": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"mempool": {
"bloom_filter_bytes": 0,
"bloom_filter_items": 0,
"bluestore_alloc_bytes": 125943744,
"bluestore_alloc_items": 8193228,
"bluestore_cache_data_bytes": 1884109,
"bluestore_cache_data_items": 829,
"bluestore_cache_onode_bytes": 28842880,
"bluestore_cache_onode_items": 42416,
"bluestore_cache_meta_bytes": 50594253,
"bluestore_cache_meta_items": 5315174,
"bluestore_cache_other_bytes": 229414948,
"bluestore_cache_other_items": 23631685,
"bluestore_Buffer_bytes": 108416,
"bluestore_Buffer_items": 968,
"bluestore_Extent_bytes": 61847760,
"bluestore_Extent_items": 1288495,
"bluestore_Blob_bytes": 140450128,
"bluestore_Blob_items": 1254019,
"bluestore_SharedBlob_bytes": 125093808,
"bluestore_SharedBlob_items": 1116909,
"bluestore_inline_bl_bytes": 369130,
"bluestore_inline_bl_items": 725,
"bluestore_fsck_bytes": 0,
"bluestore_fsck_items": 0,
"bluestore_txc_bytes": 5888,
"bluestore_txc_items": 8,
"bluestore_writing_deferred_bytes": 3317593,
"bluestore_writing_deferred_items": 794,
"bluestore_writing_bytes": 380783,
"bluestore_writing_items": 47,
"bluefs_bytes": 114472,
"bluefs_items": 6282,
"bluefs_file_reader_bytes": 705280,
"bluefs_file_reader_items": 72,
"bluefs_file_writer_bytes": 768,
"bluefs_file_writer_items": 4,
"buffer_anon_bytes": 13417560,
"buffer_anon_items": 53741,
"buffer_meta_bytes": 3034064,
"buffer_meta_items": 34478,
"osd_bytes": 1998496,
"osd_items": 173,
"osd_mapbl_bytes": 0,
"osd_mapbl_items": 0,
"osd_pglog_bytes": 205086276,
"osd_pglog_items": 814012,
"osdmap_bytes": 515864,
"osdmap_items": 11775,
"osdmap_mapping_bytes": 0,
"osdmap_mapping_items": 0,
"pgmap_bytes": 0,
"pgmap_items": 0,
"mds_co_bytes": 0,
"mds_co_items": 0,
"unittest_1_bytes": 0,
"unittest_1_items": 0,
"unittest_2_bytes": 0,
"unittest_2_items": 0
},
"objecter": {
"op_active": 0,
"op_laggy": 0,
"op_send": 0,
"op_send_bytes": 0,
"op_resend": 0,
"op_reply": 0,
"oplen_avg": {
"avgcount": 0,
"sum": 0
},
"op": 0,
"op_r": 0,
"op_w": 0,
"op_rmw": 0,
"op_pg": 0,
"osdop_stat": 0,
"osdop_create": 0,
"osdop_read": 0,
"osdop_write": 0,
"osdop_writefull": 0,
"osdop_writesame": 0,
"osdop_append": 0,
"osdop_zero": 0,
"osdop_truncate": 0,
"osdop_delete": 0,
"osdop_mapext": 0,
"osdop_sparse_read": 0,
"osdop_clonerange": 0,
"osdop_getxattr": 0,
"osdop_setxattr": 0,
"osdop_cmpxattr": 0,
"osdop_rmxattr": 0,
"osdop_resetxattrs": 0,
"osdop_call": 0,
"osdop_watch": 0,
"osdop_notify": 0,
"osdop_src_cmpxattr": 0,
"osdop_pgls": 0,
"osdop_pgls_filter": 0,
"osdop_other": 0,
"linger_active": 0,
"linger_send": 0,
"linger_resend": 0,
"linger_ping": 0,
"poolop_active": 0,
"poolop_send": 0,
"poolop_resend": 0,
"poolstat_active": 0,
"poolstat_send": 0,
"poolstat_resend": 0,
"statfs_active": 0,
"statfs_send": 0,
"statfs_resend": 0,
"command_active": 0,
"command_send": 0,
"command_resend": 0,
"map_epoch": 12426,
"map_full": 1,
"map_inc": 287,
"osd_sessions": 0,
"osd_session_open": 0,
"osd_session_close": 0,
"osd_laggy": 0,
"omap_wr": 0,
"omap_rd": 0,
"omap_del": 0
},
"osd": {
"op_wip": 0,
"op": 0,
"op_in_bytes": 0,
"op_out_bytes": 0,
"op_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_r": 0,
"op_r_out_bytes": 0,
"op_r_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_r_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_r_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_w": 0,
"op_w_in_bytes": 0,
"op_w_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_w_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_w_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_rw": 0,
"op_rw_in_bytes": 0,
"op_rw_out_bytes": 0,
"op_rw_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_rw_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_rw_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_before_queue_op_lat": {
"avgcount": 3965535,
"sum": 333.780003675,
"avgtime": 0.000084170
},
"op_before_dequeue_op_lat": {
"avgcount": 3965535,
"sum": 434.995411447,
"avgtime": 0.000109694
},
"subop": 3944307,
"subop_in_bytes": 98738982548,
"subop_latency": {
"avgcount": 3944307,
"sum": 6756.175478507,
"avgtime": 0.001712892
},
"subop_w": 3944307,
"subop_w_in_bytes": 98738982548,
"subop_w_latency": {
"avgcount": 3944307,
"sum": 6756.175478507,
"avgtime": 0.001712892
},
"subop_pull": 0,
"subop_pull_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"subop_push": 0,
"subop_push_in_bytes": 0,
"subop_push_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"pull": 0,
"push": 0,
"push_out_bytes": 0,
"recovery_ops": 0,
"recovery_bytes": 0,
"loadavg": 40,
"cached_crc": 0,
"cached_crc_adjusted": 0,
"missed_crc": 0,
"numpg": 173,
"numpg_primary": 0,
"numpg_replica": 173,
"numpg_stray": 0,
"numpg_removing": 0,
"heartbeat_to_peers": 10,
"map_messages": 2085,
"map_message_epochs": 2223,
"map_message_epoch_dups": 1935,
"messages_delayed_for_map": 0,
"osd_map_cache_hit": 50571,
"osd_map_cache_miss": 1,
"osd_map_cache_miss_low": 0,
"osd_map_cache_miss_low_avg": {
"avgcount": 0,
"sum": 0
},
"osd_map_bl_cache_hit": 39,
"osd_map_bl_cache_miss": 317,
"stat_bytes": 1920362086400,
"stat_bytes_used": 181869830144,
"stat_bytes_avail": 1738492256256,
"copyfrom": 0,
"tier_promote": 0,
"tier_flush": 0,
"tier_flush_fail": 0,
"tier_try_flush": 0,
"tier_try_flush_fail": 0,
"tier_evict": 0,
"tier_whiteout": 0,
"tier_dirty": 0,
"tier_clean": 0,
"tier_delay": 0,
"tier_proxy_read": 0,
"tier_proxy_write": 0,
"agent_wake": 0,
"agent_skip": 0,
"agent_flush": 0,
"agent_evict": 0,
"object_ctx_cache_hit": 0,
"object_ctx_cache_total": 0,
"op_cache_hit": 0,
"osd_tier_flush_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_tier_promote_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_tier_r_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_pg_info": 3979232,
"osd_pg_fastinfo": 3825183,
"osd_pg_biginfo": 33887
},
"recoverystate_perf": {
"initial_latency": {
"avgcount": 173,
"sum": 6.060385414,
"avgtime": 0.035031129
},
"started_latency": {
"avgcount": 173,
"sum": 35.126913452,
"avgtime": 0.203045742
},
"reset_latency": {
"avgcount": 346,
"sum": 696.523139606,
"avgtime": 2.013072657
},
"start_latency": {
"avgcount": 346,
"sum": 0.000675707,
"avgtime": 0.000001952
},
"primary_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"peering_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"backfilling_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"waitremotebackfillreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"waitlocalbackfillreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"notbackfilling_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"repnotrecovering_latency": {
"avgcount": 88,
"sum": 14.954158341,
"avgtime": 0.169933617
},
"repwaitrecoveryreserved_latency": {
"avgcount": 88,
"sum": 43.849329328,
"avgtime": 0.498287833
},
"repwaitbackfillreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"reprecovering_latency": {
"avgcount": 88,
"sum": 16.136443540,
"avgtime": 0.183368676
},
"activating_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"waitlocalrecoveryreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"waitremoterecoveryreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"recovering_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"recovered_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"clean_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"active_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"replicaactive_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"stray_latency": {
"avgcount": 346,
"sum": 212.848549950,
"avgtime": 0.615169219
},
"getinfo_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"getlog_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"waitactingchange_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"incomplete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"down_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"getmissing_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"waitupthru_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"notrecovering_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"rocksdb": {
"get_latency": {
"avgcount": 790064,
"sum": 51.480388766,
"avgtime": 0.000065159
},
"submit_latency": {
"avgcount": 3979669,
"sum": 377.602827079,
"avgtime": 0.000094882
},
"submit_sync_latency": {
"avgcount": 3291698,
"sum": 1909.117762596,
"avgtime": 0.000579979
},
"compact": 0,
"compact_range": 0,
"compact_queue_merge": 0,
"compact_queue_len": 0,
"rocksdb_write_wal_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_memtable_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_delay_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_pre_and_post_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
}
}
We monitor osd.op_w_latency to check for problems on our clusters. Does someone know how to fix this ?
Best regards,
Ingénieur Système et Réseau
Fixe : +33 (0)3 59 82 20 20
Cyllene Lille –
393 rue du général
de Gaulle – 59700 Marcq-en-baroeul
www.groupe-cyllene.com |
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
