hi igor, hi paul - thank you for your answers. On 5/19/20 2:05 PM, Igor Fedotov wrote: > I presume that your OSDs suffer from slow RocksDB access, > collection_listing operation is a culprit in this case - 30 items > listing takes 96seconds to complete. > From my experience such issues tend to happen after massive DB data > removals (e.g. pool removal(s)) often backed by RGW usage which is "DB > access greedy". > DB data fragmentation is presumably the root cause for the resulting > slowdown. BlueFS spillover to main HDD device if any to be eliminated too. > To temporary workaround the issue you might want to do manual RocksDB > compaction - it's known to be helpful in such cases. But the positive > effect doesn't last forever - DB might go into degraded state again. so i'll try to compact the rockdbs and report back … we didn't see any spillovers yet, but indeed created a few large test pools with many pgs and removed these afterwards. also, the affected pools had a significant number of osds added to them recently. apart from this, the pools are mainly being used for cephfs, with some rather small rgw pools for openstack on top. On 5/19/20 2:13 PM, Paul Emmerich wrote: > 3) if necessary add more OSDs; common problem is having very > few dedicated OSDs for the index pool; running the index on > all OSDs (and having a fast DB device for every disk) is > better. But sounds like you already have that nope, unfortunately not. default.rgw.buckets.index is an replicated pool on hdds with only 4 pgs, i'll see if i can change that. back to igors questions: > Some questions about your cases: > - What kind of payload do you have - RGW or something else? mostly cephfs. the most active pools in terms of i/o are the openstack rgw ones, though. > - Have you done massive removals recently? yes, see above > - How large are main and DB disks for suffering OSDs? How much is their > current utilization? for osd.293, for which i've sent the log: main: 2tb hdd (5% used), db: 14gb partition on a 180gb nvme (~400mb used) … i'll attach a perf dump for this osd. > - Do you see multiple "slow operation observed" patterns in OSD logs? yes, although they do not necessarily correlate with osd down events. > Are they all about _collection_list function? no, there are also submit_transact and _txc_committed_kv, with about the same frequency as collection_list. thank you very much for your analysis & with kind regards, thoralf.
{
"AsyncMessenger::Worker-0": {
"msgr_recv_messages": 370054,
"msgr_send_messages": 358503,
"msgr_recv_bytes": 4872205873,
"msgr_send_bytes": 777758278,
"msgr_created_connections": 1093,
"msgr_active_connections": 120,
"msgr_running_total_time": 47.973738728,
"msgr_running_send_time": 17.013489976,
"msgr_running_recv_time": 15.136124921,
"msgr_running_fast_dispatch_time": 8.439584980
},
"AsyncMessenger::Worker-1": {
"msgr_recv_messages": 425683,
"msgr_send_messages": 426785,
"msgr_recv_bytes": 932978303,
"msgr_send_bytes": 8325976667,
"msgr_created_connections": 1166,
"msgr_active_connections": 135,
"msgr_running_total_time": 56.763030213,
"msgr_running_send_time": 26.229444997,
"msgr_running_recv_time": 10.858542062,
"msgr_running_fast_dispatch_time": 9.659598692
},
"AsyncMessenger::Worker-2": {
"msgr_recv_messages": 718122,
"msgr_send_messages": 713075,
"msgr_recv_bytes": 1384486805,
"msgr_send_bytes": 7955598259,
"msgr_created_connections": 1177,
"msgr_active_connections": 206,
"msgr_running_total_time": 88.454218976,
"msgr_running_send_time": 39.950859233,
"msgr_running_recv_time": 19.314262319,
"msgr_running_fast_dispatch_time": 14.587312053
},
"bluefs": {
"gift_bytes": 0,
"reclaim_bytes": 0,
"db_total_bytes": 15032377344,
"db_used_bytes": 411033600,
"wal_total_bytes": 0,
"wal_used_bytes": 0,
"slow_total_bytes": 94737203200,
"slow_used_bytes": 10714480640,
"num_files": 190,
"log_bytes": 40480768,
"log_compactions": 0,
"logged_bytes": 39432192,
"files_written_wal": 1,
"files_written_sst": 11,
"bytes_written_wal": 113707509,
"bytes_written_sst": 645429643,
"bytes_written_slow": 69230592,
"max_bytes_wal": 0,
"max_bytes_db": 833609728,
"max_bytes_slow": 10714480640,
"read_random_count": 57142829,
"read_random_bytes": 235468132466,
"read_random_disk_count": 2141778,
"read_random_disk_bytes": 10113139075,
"read_random_buffer_count": 55026965,
"read_random_buffer_bytes": 225354993391,
"read_count": 969858,
"read_bytes": 248129279117,
"read_prefetch_count": 964823,
"read_prefetch_bytes": 247963616292
},
"bluestore": {
"kv_flush_lat": {
"avgcount": 9608,
"sum": 0.029595609,
"avgtime": 0.000003080
},
"kv_commit_lat": {
"avgcount": 9608,
"sum": 11.087871327,
"avgtime": 0.001154024
},
"kv_sync_lat": {
"avgcount": 9608,
"sum": 11.117466936,
"avgtime": 0.001157105
},
"kv_final_lat": {
"avgcount": 9574,
"sum": 0.814363631,
"avgtime": 0.000085059
},
"state_prepare_lat": {
"avgcount": 11192,
"sum": 131.870915861,
"avgtime": 0.011782605
},
"state_aio_wait_lat": {
"avgcount": 11192,
"sum": 67.730525910,
"avgtime": 0.006051691
},
"state_io_done_lat": {
"avgcount": 11192,
"sum": 0.603595585,
"avgtime": 0.000053930
},
"state_kv_queued_lat": {
"avgcount": 11192,
"sum": 3.042294156,
"avgtime": 0.000271827
},
"state_kv_commiting_lat": {
"avgcount": 11192,
"sum": 13.306189747,
"avgtime": 0.001188901
},
"state_kv_done_lat": {
"avgcount": 11192,
"sum": 0.010666506,
"avgtime": 0.000000953
},
"state_deferred_queued_lat": {
"avgcount": 3263,
"sum": 163317.001772899,
"avgtime": 50.051180439
},
"state_deferred_aio_wait_lat": {
"avgcount": 3263,
"sum": 327.452951142,
"avgtime": 0.100353340
},
"state_deferred_cleanup_lat": {
"avgcount": 3263,
"sum": 10816.952063442,
"avgtime": 3.315032811
},
"state_finishing_lat": {
"avgcount": 11175,
"sum": 0.002737840,
"avgtime": 0.000000244
},
"state_done_lat": {
"avgcount": 11175,
"sum": 1734.748612643,
"avgtime": 0.155234775
},
"throttle_lat": {
"avgcount": 11192,
"sum": 0.312166152,
"avgtime": 0.000027891
},
"submit_lat": {
"avgcount": 11192,
"sum": 132.222524889,
"avgtime": 0.011814021
},
"commit_lat": {
"avgcount": 11192,
"sum": 216.557994648,
"avgtime": 0.019349356
},
"read_lat": {
"avgcount": 36943,
"sum": 174.708161031,
"avgtime": 0.004729127
},
"read_onode_meta_lat": {
"avgcount": 74403,
"sum": 2.702055271,
"avgtime": 0.000036316
},
"read_wait_aio_lat": {
"avgcount": 37460,
"sum": 168.306482143,
"avgtime": 0.004492965
},
"compress_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"decompress_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"csum_lat": {
"avgcount": 47455,
"sum": 3.641494642,
"avgtime": 0.000076735
},
"compress_success_count": 0,
"compress_rejected_count": 0,
"write_pad_bytes": 20961338,
"deferred_write_ops": 4237,
"deferred_write_bytes": 35360768,
"write_penalty_read_ops": 2187,
"bluestore_allocated": 36155817984,
"bluestore_stored": 23339449340,
"bluestore_compressed": 0,
"bluestore_compressed_allocated": 0,
"bluestore_compressed_original": 0,
"bluestore_onodes": 38563,
"bluestore_onode_hits": 320261,
"bluestore_onode_misses": 75966,
"bluestore_onode_shard_hits": 19126,
"bluestore_onode_shard_misses": 10034,
"bluestore_extents": 38571,
"bluestore_blobs": 38689,
"bluestore_buffers": 2957,
"bluestore_buffer_bytes": 263360512,
"bluestore_buffer_hit_bytes": 850723509,
"bluestore_buffer_miss_bytes": 9129609535,
"bluestore_write_big": 2452,
"bluestore_write_big_bytes": 1031536640,
"bluestore_write_big_blobs": 2757,
"bluestore_write_small": 10038,
"bluestore_write_small_bytes": 97910028,
"bluestore_write_small_unused": 0,
"bluestore_write_small_deferred": 8520,
"bluestore_write_small_pre_read": 4510,
"bluestore_write_small_new": 4275,
"bluestore_txc": 11192,
"bluestore_onode_reshard": 104,
"bluestore_blob_split": 0,
"bluestore_extent_compress": 2187,
"bluestore_gc_merged": 0,
"bluestore_read_eio": 0,
"bluestore_reads_with_retries": 0,
"bluestore_fragmentation_micros": 30,
"omap_seek_to_first_lat": {
"avgcount": 76,
"sum": 0.000490927,
"avgtime": 0.000006459
},
"omap_upper_bound_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"omap_lower_bound_lat": {
"avgcount": 766,
"sum": 0.004993627,
"avgtime": 0.000006519
},
"omap_next_lat": {
"avgcount": 227755,
"sum": 0.340256760,
"avgtime": 0.000001493
},
"clist_lat": {
"avgcount": 332,
"sum": 578.220620331,
"avgtime": 1.741628374
}
},
"cct": {
"total_workers": 6,
"unhealthy_workers": 0
},
"finisher-commit_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 2184,
"sum": 2.156908144,
"avgtime": 0.000987595
}
},
"finisher-defered_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"finisher-objecter-finisher-0": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"mempool": {
"bloom_filter_bytes": 0,
"bloom_filter_items": 0,
"bluestore_alloc_bytes": 7681288,
"bluestore_alloc_items": 960161,
"bluestore_cache_data_bytes": 263307264,
"bluestore_cache_data_items": 3265,
"bluestore_cache_onode_bytes": 25296016,
"bluestore_cache_onode_items": 38561,
"bluestore_cache_other_bytes": 54243524,
"bluestore_cache_other_items": 2523785,
"bluestore_fsck_bytes": 0,
"bluestore_fsck_items": 0,
"bluestore_txc_bytes": 11648,
"bluestore_txc_items": 16,
"bluestore_writing_deferred_bytes": 1065419,
"bluestore_writing_deferred_items": 251,
"bluestore_writing_bytes": 5391808,
"bluestore_writing_items": 1514,
"bluefs_bytes": 130304,
"bluefs_items": 4398,
"buffer_anon_bytes": 50340492,
"buffer_anon_items": 3272,
"buffer_meta_bytes": 419848,
"buffer_meta_items": 4771,
"osd_bytes": 963184,
"osd_items": 74,
"osd_mapbl_bytes": 17952905,
"osd_mapbl_items": 50,
"osd_pglog_bytes": 149124918,
"osd_pglog_items": 439896,
"osdmap_bytes": 8491320,
"osdmap_items": 288467,
"osdmap_mapping_bytes": 0,
"osdmap_mapping_items": 0,
"pgmap_bytes": 0,
"pgmap_items": 0,
"mds_co_bytes": 0,
"mds_co_items": 0,
"unittest_1_bytes": 0,
"unittest_1_items": 0,
"unittest_2_bytes": 0,
"unittest_2_items": 0
},
"objecter": {
"op_active": 0,
"op_laggy": 0,
"op_send": 0,
"op_send_bytes": 0,
"op_resend": 0,
"op_reply": 0,
"op": 0,
"op_r": 0,
"op_w": 0,
"op_rmw": 0,
"op_pg": 0,
"osdop_stat": 0,
"osdop_create": 0,
"osdop_read": 0,
"osdop_write": 0,
"osdop_writefull": 0,
"osdop_writesame": 0,
"osdop_append": 0,
"osdop_zero": 0,
"osdop_truncate": 0,
"osdop_delete": 0,
"osdop_mapext": 0,
"osdop_sparse_read": 0,
"osdop_clonerange": 0,
"osdop_getxattr": 0,
"osdop_setxattr": 0,
"osdop_cmpxattr": 0,
"osdop_rmxattr": 0,
"osdop_resetxattrs": 0,
"osdop_call": 0,
"osdop_watch": 0,
"osdop_notify": 0,
"osdop_src_cmpxattr": 0,
"osdop_pgls": 0,
"osdop_pgls_filter": 0,
"osdop_other": 0,
"linger_active": 0,
"linger_send": 0,
"linger_resend": 0,
"linger_ping": 0,
"poolop_active": 0,
"poolop_send": 0,
"poolop_resend": 0,
"poolstat_active": 0,
"poolstat_send": 0,
"poolstat_resend": 0,
"statfs_active": 0,
"statfs_send": 0,
"statfs_resend": 0,
"command_active": 0,
"command_send": 0,
"command_resend": 0,
"map_epoch": 648915,
"map_full": 0,
"map_inc": 2255,
"osd_sessions": 0,
"osd_session_open": 0,
"osd_session_close": 0,
"osd_laggy": 0,
"omap_wr": 0,
"omap_rd": 0,
"omap_del": 0
},
"osd": {
"op_wip": 0,
"op": 0,
"op_in_bytes": 0,
"op_out_bytes": 0,
"op_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_r": 0,
"op_r_out_bytes": 0,
"op_r_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_r_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_r_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_w": 0,
"op_w_in_bytes": 0,
"op_w_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_w_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_w_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_rw": 0,
"op_rw_in_bytes": 0,
"op_rw_out_bytes": 0,
"op_rw_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_rw_process_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_rw_prepare_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"op_before_queue_op_lat": {
"avgcount": 223942,
"sum": 5.726237872,
"avgtime": 0.000025570
},
"op_before_dequeue_op_lat": {
"avgcount": 224007,
"sum": 161.880825771,
"avgtime": 0.000722659
},
"subop": 0,
"subop_in_bytes": 0,
"subop_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"subop_w": 0,
"subop_w_in_bytes": 0,
"subop_w_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"subop_pull": 0,
"subop_pull_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"subop_push": 0,
"subop_push_in_bytes": 0,
"subop_push_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"pull": 0,
"push": 766,
"push_out_bytes": 6342967534,
"recovery_ops": 54070,
"recovery_bytes": 6342967570,
"loadavg": 306,
"cached_crc": 0,
"cached_crc_adjusted": 0,
"missed_crc": 0,
"numpg": 74,
"numpg_primary": 11,
"numpg_replica": 56,
"numpg_stray": 7,
"numpg_removing": 0,
"heartbeat_to_peers": 70,
"map_messages": 28485,
"map_message_epochs": 30586,
"map_message_epoch_dups": 33400,
"messages_delayed_for_map": 0,
"osd_map_cache_hit": 184563,
"osd_map_cache_miss": 22,
"osd_map_cache_miss_low": 0,
"osd_map_cache_miss_low_avg": {
"avgcount": 0,
"sum": 0
},
"osd_map_bl_cache_hit": 17404,
"osd_map_bl_cache_miss": 4456,
"stat_bytes": 2015430172672,
"stat_bytes_used": 61902282752,
"stat_bytes_avail": 1953527889920,
"copyfrom": 0,
"tier_promote": 0,
"tier_flush": 0,
"tier_flush_fail": 0,
"tier_try_flush": 0,
"tier_try_flush_fail": 0,
"tier_evict": 0,
"tier_whiteout": 0,
"tier_dirty": 0,
"tier_clean": 0,
"tier_delay": 0,
"tier_proxy_read": 0,
"tier_proxy_write": 0,
"agent_wake": 0,
"agent_skip": 0,
"agent_flush": 0,
"agent_evict": 0,
"object_ctx_cache_hit": 53261,
"object_ctx_cache_total": 107324,
"op_cache_hit": 0,
"osd_tier_flush_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_tier_promote_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_tier_r_lat": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"osd_pg_info": 8693,
"osd_pg_fastinfo": 0,
"osd_pg_biginfo": 1366
},
"prioritycache": {
"target_bytes": 3221225472,
"mapped_bytes": 2575081472,
"unmapped_bytes": 170033152,
"heap_bytes": 2745114624,
"cache_bytes": 1932735282
},
"prioritycache:data": {
"pri0_bytes": 0,
"pri1_bytes": 263360512,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 0,
"reserved_bytes": 139292672,
"committed_bytes": 402653184
},
"prioritycache:kv": {
"pri0_bytes": 95911760,
"pri1_bytes": 1066099406,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 0,
"reserved_bytes": 138223074,
"committed_bytes": 1300234240
},
"prioritycache:meta": {
"pri0_bytes": 0,
"pri1_bytes": 79544596,
"pri2_bytes": 0,
"pri3_bytes": 0,
"pri4_bytes": 0,
"pri5_bytes": 0,
"pri6_bytes": 0,
"pri7_bytes": 0,
"pri8_bytes": 0,
"pri9_bytes": 0,
"pri10_bytes": 0,
"pri11_bytes": 0,
"reserved_bytes": 138559212,
"committed_bytes": 218103808
},
"recoverystate_perf": {
"initial_latency": {
"avgcount": 95,
"sum": 1.639823247,
"avgtime": 0.017261297
},
"started_latency": {
"avgcount": 585,
"sum": 449149.966363433,
"avgtime": 767.777720279
},
"reset_latency": {
"avgcount": 680,
"sum": 142.649119899,
"avgtime": 0.209778117
},
"start_latency": {
"avgcount": 680,
"sum": 0.050965587,
"avgtime": 0.000074949
},
"primary_latency": {
"avgcount": 80,
"sum": 86980.010451169,
"avgtime": 1087.250130639
},
"peering_latency": {
"avgcount": 80,
"sum": 180.897855962,
"avgtime": 2.261223199
},
"backfilling_latency": {
"avgcount": 11,
"sum": 2315.292763778,
"avgtime": 210.481160343
},
"waitremotebackfillreserved_latency": {
"avgcount": 23,
"sum": 22139.617318975,
"avgtime": 962.592057346
},
"waitlocalbackfillreserved_latency": {
"avgcount": 43,
"sum": 9241.558169898,
"avgtime": 214.919957439
},
"notbackfilling_latency": {
"avgcount": 4,
"sum": 0.002889054,
"avgtime": 0.000722263
},
"repnotrecovering_latency": {
"avgcount": 298,
"sum": 354946.516065738,
"avgtime": 1191.095691495
},
"repwaitrecoveryreserved_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"repwaitbackfillreserved_latency": {
"avgcount": 27,
"sum": 242.795322429,
"avgtime": 8.992419349
},
"reprecovering_latency": {
"avgcount": 20,
"sum": 401.685752811,
"avgtime": 20.084287640
},
"activating_latency": {
"avgcount": 66,
"sum": 11.764009769,
"avgtime": 0.178242572
},
"waitlocalrecoveryreserved_latency": {
"avgcount": 2,
"sum": 0.001285103,
"avgtime": 0.000642551
},
"waitremoterecoveryreserved_latency": {
"avgcount": 2,
"sum": 0.003136568,
"avgtime": 0.001568284
},
"recovering_latency": {
"avgcount": 2,
"sum": 180.461610956,
"avgtime": 90.230805478
},
"recovered_latency": {
"avgcount": 33,
"sum": 1.440625492,
"avgtime": 0.043655317
},
"clean_latency": {
"avgcount": 24,
"sum": 53191.885256114,
"avgtime": 2216.328552338
},
"active_latency": {
"avgcount": 55,
"sum": 86785.647086614,
"avgtime": 1577.920856120
},
"replicaactive_latency": {
"avgcount": 279,
"sum": 355591.014860010,
"avgtime": 1274.519766523
},
"stray_latency": {
"avgcount": 594,
"sum": 6834.653349816,
"avgtime": 11.506150420
},
"getinfo_latency": {
"avgcount": 80,
"sum": 64.438126744,
"avgtime": 0.805476584
},
"getlog_latency": {
"avgcount": 77,
"sum": 1.292891397,
"avgtime": 0.016790797
},
"waitactingchange_latency": {
"avgcount": 11,
"sum": 26.024044356,
"avgtime": 2.365822214
},
"incomplete_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"down_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"getmissing_latency": {
"avgcount": 66,
"sum": 0.004620661,
"avgtime": 0.000070010
},
"waitupthru_latency": {
"avgcount": 64,
"sum": 115.156561738,
"avgtime": 1.799321277
},
"notrecovering_latency": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"rocksdb": {
"get": 88147,
"submit_transaction": 11192,
"submit_transaction_sync": 9608,
"get_latency": {
"avgcount": 88147,
"sum": 4.113963412,
"avgtime": 0.000046671
},
"submit_latency": {
"avgcount": 11192,
"sum": 1.875353195,
"avgtime": 0.000167561
},
"submit_sync_latency": {
"avgcount": 9608,
"sum": 8.942942103,
"avgtime": 0.000930780
},
"compact": 0,
"compact_range": 0,
"compact_queue_merge": 0,
"compact_queue_len": 0,
"rocksdb_write_wal_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_memtable_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_delay_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
},
"rocksdb_write_pre_and_post_time": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-bluestore_throttle_bytes": {
"val": 0,
"max": 67108864,
"get_started": 11192,
"get": 11192,
"get_sum": 16765336334,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 9608,
"put_sum": 16765336334,
"wait": {
"avgcount": 1,
"sum": 0.300348079,
"avgtime": 0.300348079
}
},
"throttle-bluestore_throttle_deferred_bytes": {
"val": 47001477,
"max": 201326592,
"get_started": 0,
"get": 3280,
"get_sum": 9647425931,
"get_or_fail_fail": 0,
"get_or_fail_success": 3280,
"take": 0,
"take_sum": 0,
"put": 223,
"put_sum": 9600424454,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-client": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 241,
"get_sum": 6039429,
"get_or_fail_fail": 0,
"get_or_fail_success": 241,
"take": 0,
"take_sum": 0,
"put": 241,
"put_sum": 6039429,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-cluster": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 331207,
"get_sum": 4702068003,
"get_or_fail_fail": 0,
"get_or_fail_success": 331207,
"take": 0,
"take_sum": 0,
"put": 331207,
"put_sum": 4702068003,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_back_client": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 306580,
"get_sum": 614386320,
"get_or_fail_fail": 0,
"get_or_fail_success": 306580,
"take": 0,
"take_sum": 0,
"put": 306580,
"put_sum": 614386320,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_back_server": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 284622,
"get_sum": 570382488,
"get_or_fail_fail": 0,
"get_or_fail_success": 284622,
"take": 0,
"take_sum": 0,
"put": 284622,
"put_sum": 570382488,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_front_client": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 306580,
"get_sum": 614386320,
"get_or_fail_fail": 0,
"get_or_fail_success": 306580,
"take": 0,
"take_sum": 0,
"put": 306580,
"put_sum": 614386320,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-hb_front_server": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 284621,
"get_sum": 570380484,
"get_or_fail_fail": 0,
"get_or_fail_success": 284621,
"take": 0,
"take_sum": 0,
"put": 284621,
"put_sum": 570380484,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-msgr_dispatch_throttler-ms_objecter": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-objecter_bytes": {
"val": 0,
"max": 104857600,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-objecter_ops": {
"val": 0,
"max": 1024,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
},
"throttle-osd_client_bytes": {
"val": 0,
"max": 524288000,
"get_started": 0,
"get": 0,
"get_sum": 0,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 0,
"put_sum": 0,
"wait": {
"avgcount": 0,
"sum": 0.000000000,
"avgtime": 0.000000000
}
}
}
Attachment:
signature.asc
Description: OpenPGP digital signature
_______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
