Here you have. Perf dump: { "AsyncMessenger::Worker-0": { "msgr_recv_messages": 12239872, "msgr_send_messages": 12284221, "msgr_recv_bytes": 43759275160, "msgr_send_bytes": 61268769426, "msgr_created_connections": 754, "msgr_active_connections": 100, "msgr_running_total_time": 939.476931816, "msgr_running_send_time": 337.873686715, "msgr_running_recv_time": 360.728238752, "msgr_running_fast_dispatch_time": 183.737116872, "msgr_send_messages_queue_lat": { "avgcount": 12284206, "sum": 1538.989479364, "avgtime": 0.000125281 }, "msgr_handle_ack_lat": { "avgcount": 5258403, "sum": 1.005075918, "avgtime": 0.000000191 } }, "AsyncMessenger::Worker-1": { "msgr_recv_messages": 12099771, "msgr_send_messages": 12138795, "msgr_recv_bytes": 56967534605, "msgr_send_bytes": 130548664272, "msgr_created_connections": 647, "msgr_active_connections": 91, "msgr_running_total_time": 977.277996439, "msgr_running_send_time": 362.155959231, "msgr_running_recv_time": 365.376281473, "msgr_running_fast_dispatch_time": 191.186643292, "msgr_send_messages_queue_lat": { "avgcount": 12138818, "sum": 1557.187685700, "avgtime": 0.000128281 }, "msgr_handle_ack_lat": { "avgcount": 6155265, "sum": 1.096270527, "avgtime": 0.000000178 } }, "AsyncMessenger::Worker-2": { "msgr_recv_messages": 11858354, "msgr_send_messages": 11960404, "msgr_recv_bytes": 60727084610, "msgr_send_bytes": 168534726650, "msgr_created_connections": 1043, "msgr_active_connections": 103, "msgr_running_total_time": 937.324084772, "msgr_running_send_time": 351.174710644, "msgr_running_recv_time": 2744.276782474, "msgr_running_fast_dispatch_time": 172.960322050, "msgr_send_messages_queue_lat": { "avgcount": 11960392, "sum": 1763.762581924, "avgtime": 0.000147466 }, "msgr_handle_ack_lat": { "avgcount": 2651457, "sum": 0.538495450, "avgtime": 0.000000203 } }, "bluefs": { "db_total_bytes": 128005955584, "db_used_bytes": 3271557120, "wal_total_bytes": 0, "wal_used_bytes": 0, "slow_total_bytes": 10000810377216, "slow_used_bytes": 0, "num_files": 70, "log_bytes": 13045760, "log_compactions": 58, "logged_bytes": 922333184, "files_written_wal": 2, "files_written_sst": 13, "bytes_written_wal": 1988489216, "bytes_written_sst": 268890112, "bytes_written_slow": 0, "max_bytes_wal": 0, "max_bytes_db": 3271557120, "max_bytes_slow": 0, "read_random_count": 577484, "read_random_bytes": 2879541532, "read_random_disk_count": 284290, "read_random_disk_bytes": 1540394118, "read_random_buffer_count": 319088, "read_random_buffer_bytes": 1339147414, "read_count": 1086625, "read_bytes": 15054317429, "read_prefetch_count": 1069462, "read_prefetch_bytes": 14506469332, "read_zeros_candidate": 0, "read_zeros_errors": 0 }, "bluestore": { "kv_flush_lat": { "avgcount": 225099, "sum": 526.605165277, "avgtime": 0.002339438 }, "kv_commit_lat": { "avgcount": 225099, "sum": 61.412175620, "avgtime": 0.000272822 }, "kv_sync_lat": { "avgcount": 225099, "sum": 588.017340897, "avgtime": 0.002612261 }, "kv_final_lat": { "avgcount": 225096, "sum": 6.516869320, "avgtime": 0.000028951 }, "state_prepare_lat": { "avgcount": 241063, "sum": 173.705759592, "avgtime": 0.000720582 }, "state_aio_wait_lat": { "avgcount": 241063, "sum": 1008.936150524, "avgtime": 0.004185362 }, "state_io_done_lat": { "avgcount": 241063, "sum": 2.923457351, "avgtime": 0.000012127 }, "state_kv_queued_lat": { "avgcount": 241063, "sum": 560.050193021, "avgtime": 0.002323252 }, "state_kv_commiting_lat": { "avgcount": 241063, "sum": 68.355225981, "avgtime": 0.000283557 }, "state_kv_done_lat": { "avgcount": 241063, "sum": 0.097836444, "avgtime": 0.000000405 }, "state_deferred_queued_lat": { "avgcount": 47230, "sum": 70711.960011607, "avgtime": 1.497183146 }, "state_deferred_aio_wait_lat": { "avgcount": 47230, "sum": 70.931594687, "avgtime": 0.001501833 }, "state_deferred_cleanup_lat": { "avgcount": 47229, "sum": 333702.384485631, "avgtime": 7.065624605 }, "state_finishing_lat": { "avgcount": 241062, "sum": 0.033475853, "avgtime": 0.000000138 }, "state_done_lat": { "avgcount": 241062, "sum": 15258.375631838, "avgtime": 0.063296478 }, "throttle_lat": { "avgcount": 241063, "sum": 67.464167288, "avgtime": 0.000279861 }, "submit_lat": { "avgcount": 241063, "sum": 190.468701838, "avgtime": 0.000790120 }, "commit_lat": { "avgcount": 241063, "sum": 1814.026687331, "avgtime": 0.007525114 }, "read_lat": { "avgcount": 1590088, "sum": 8928.843945649, "avgtime": 0.005615314 }, "read_onode_meta_lat": { "avgcount": 3106127, "sum": 18.873640476, "avgtime": 0.000006076 }, "read_wait_aio_lat": { "avgcount": 1516039, "sum": 8830.114048334, "avgtime": 0.005824463 }, "compress_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "decompress_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "csum_lat": { "avgcount": 7047778, "sum": 52.140862362, "avgtime": 0.000007398 }, "compress_success_count": 0, "compress_rejected_count": 0, "write_pad_bytes": 194870362, "deferred_write_ops": 50669, "deferred_write_bytes": 388481024, "write_penalty_read_ops": 2067, "bluestore_allocated": 294756155392, "bluestore_stored": 292495538258, "bluestore_compressed": 0, "bluestore_compressed_allocated": 0, "bluestore_compressed_original": 0, "bluestore_onodes": 266002, "bluestore_pinned_onodes": 1, "onode_hits": 8782379, "onode_misses": 6047682, "onode_shard_hits": 1277688, "bluestore_onode_shard_misses": 947296, "bluestore_extents": 840549, "bluestore_blobs": 840549, "bluestore_buffers": 3591, "bluestore_buffer_bytes": 224182272, "bluestore_buffer_hit_bytes": 110213431882, "bluestore_buffer_miss_bytes": 406742926152, "bluestore_write_big": 39721, "bluestore_write_big_bytes": 94625427456, "bluestore_write_big_blobs": 1450312, "bluestore_write_big_deferred": 0, "bluestore_write_small": 60521, "bluestore_write_small_bytes": 48842873, "bluestore_write_small_unused": 926, "bluestore_write_deferred": 62388, "bluestore_write_deferred_bytes": 398618624, "bluestore_write_small_pre_read": 7434, "bluestore_write_new": 1448445, "bluestore_txc": 241063, "bluestore_onode_reshard": 27825, "bluestore_blob_split": 5, "bluestore_extent_compress": 3758, "bluestore_gc_merged": 0, "bluestore_read_eio": 0, "bluestore_reads_with_retries": 0, "bluestore_fragmentation_micros": 0, "omap_seek_to_first_lat": { "avgcount": 759176, "sum": 8.351645295, "avgtime": 0.000011000 }, "omap_upper_bound_lat": { "avgcount": 308, "sum": 0.002216099, "avgtime": 0.000007195 }, "omap_lower_bound_lat": { "avgcount": 87154, "sum": 1.192486653, "avgtime": 0.000013682 }, "omap_next_lat": { "avgcount": 308598, "sum": 0.211502859, "avgtime": 0.000000685 }, "omap_get_keys_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "omap_get_values_lat": { "avgcount": 19348, "sum": 0.032918565, "avgtime": 0.000001701 }, "clist_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "remove_lat": { "avgcount": 239512, "sum": 121.690074965, "avgtime": 0.000508075 } }, "bluestore-pricache": { "target_bytes": 6713193267, "mapped_bytes": 6718742528, "unmapped_bytes": 467025920, "heap_bytes": 7185768448, "cache_bytes": 4161537138 }, "bluestore-pricache:data": { "pri0_bytes": 0, "pri1_bytes": 224182272, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 30742099, "reserved_bytes": 80619949, "committed_bytes": 335544320 }, "bluestore-pricache:kv": { "pri0_bytes": 0, "pri1_bytes": 176540224, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 230565745, "reserved_bytes": 79433295, "committed_bytes": 486539264 }, "bluestore-pricache:kv_onode": { "pri0_bytes": 0, "pri1_bytes": 1977148816, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 20494732, "reserved_bytes": 82731236, "committed_bytes": 2080374784 }, "bluestore-pricache:meta": { "pri0_bytes": 0, "pri1_bytes": 942217320, "pri2_bytes": 0, "pri3_bytes": 0, "pri4_bytes": 0, "pri5_bytes": 0, "pri6_bytes": 0, "pri7_bytes": 0, "pri8_bytes": 0, "pri9_bytes": 0, "pri10_bytes": 0, "pri11_bytes": 230565745, "reserved_bytes": 68730919, "committed_bytes": 1241513984 }, "cct": { "total_workers": 5, "unhealthy_workers": 0 }, "finisher-commit_finisher": { "queue_len": 0, "complete_latency": { "avgcount": 1507, "sum": 0.568110372, "avgtime": 0.000376981 } }, "finisher-objecter-finisher-0": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluestore_alloc_bytes": 12461120, "bluestore_alloc_items": 155764, "bluestore_cache_data_bytes": 233998707, "bluestore_cache_data_items": 229226, "bluestore_cache_onode_bytes": 163857232, "bluestore_cache_onode_items": 266002, "bluestore_cache_meta_bytes": 456521298, "bluestore_cache_meta_items": 84112717, "bluestore_cache_other_bytes": 91560856, "bluestore_cache_other_items": 11308804, "bluestore_Buffer_bytes": 513888, "bluestore_Buffer_items": 5353, "bluestore_Extent_bytes": 40346352, "bluestore_Extent_items": 840549, "bluestore_Blob_bytes": 94141488, "bluestore_Blob_items": 840549, "bluestore_SharedBlob_bytes": 94141488, "bluestore_SharedBlob_items": 840549, "bluestore_inline_bl_bytes": 1134718, "bluestore_inline_bl_items": 8758, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_txc_bytes": 784, "bluestore_txc_items": 1, "bluestore_writing_deferred_bytes": 1918388, "bluestore_writing_deferred_items": 81, "bluestore_writing_bytes": 0, "bluestore_writing_items": 0, "bluefs_bytes": 36800, "bluefs_items": 1443, "bluefs_file_reader_bytes": 1842688, "bluefs_file_reader_items": 95, "bluefs_file_writer_bytes": 576, "bluefs_file_writer_items": 3, "buffer_anon_bytes": 10746933, "buffer_anon_items": 25967, "buffer_meta_bytes": 2541352, "buffer_meta_items": 28879, "osd_bytes": 916272, "osd_items": 81, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 164733192, "osd_pglog_items": 363565, "osdmap_bytes": 2182392, "osdmap_items": 53839, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "mds_co_bytes": 0, "mds_co_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 0, "op_send_bytes": 0, "op_resend": 0, "op_reply": 0, "oplen_avg": { "avgcount": 0, "sum": 0 }, "op": 0, "op_r": 0, "op_w": 0, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 63038, "map_full": 2, "map_inc": 1511, "osd_sessions": 0, "osd_session_open": 0, "osd_session_close": 0, "osd_laggy": 0, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "osd": { "op_wip": 0, "op": 272274, "op_in_bytes": 39148504708, "op_out_bytes": 181035613199, "op_latency": { "avgcount": 272274, "sum": 2312.558437554, "avgtime": 0.008493497 }, "op_process_latency": { "avgcount": 272274, "sum": 2151.138778914, "avgtime": 0.007900639 }, "op_prepare_latency": { "avgcount": 280169, "sum": 1068.077633101, "avgtime": 0.003812262 }, "op_r": 191667, "op_r_out_bytes": 181035613199, "op_r_latency": { "avgcount": 191667, "sum": 1035.691613777, "avgtime": 0.005403599 }, "op_r_process_latency": { "avgcount": 191667, "sum": 959.303890513, "avgtime": 0.005005055 }, "op_r_prepare_latency": { "avgcount": 191712, "sum": 962.370649266, "avgtime": 0.005019876 }, "op_w": 34260, "op_w_in_bytes": 39121651122, "op_w_latency": { "avgcount": 34260, "sum": 1117.333684176, "avgtime": 0.032613359 }, "op_w_process_latency": { "avgcount": 34260, "sum": 1059.202697781, "avgtime": 0.030916599 }, "op_w_prepare_latency": { "avgcount": 36798, "sum": 91.420473715, "avgtime": 0.002484387 }, "op_rw": 46347, "op_rw_in_bytes": 26853586, "op_rw_out_bytes": 0, "op_rw_latency": { "avgcount": 46347, "sum": 159.533139601, "avgtime": 0.003442145 }, "op_rw_process_latency": { "avgcount": 46347, "sum": 132.632190620, "avgtime": 0.002861721 }, "op_rw_prepare_latency": { "avgcount": 51659, "sum": 14.286510120, "avgtime": 0.000276554 }, "op_before_queue_op_lat": { "avgcount": 897996, "sum": 154.309110719, "avgtime": 0.000171837 }, "op_before_dequeue_op_lat": { "avgcount": 901696, "sum": 622.443106332, "avgtime": 0.000690302 }, "subop": 119722, "subop_in_bytes": 55578084121, "subop_latency": { "avgcount": 119722, "sum": 1159.173264760, "avgtime": 0.009682207 }, "subop_w": 119722, "subop_w_in_bytes": 55578084121, "subop_w_latency": { "avgcount": 119722, "sum": 1159.173264760, "avgtime": 0.009682207 }, "subop_pull": 0, "subop_pull_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "subop_push": 0, "subop_push_in_bytes": 0, "subop_push_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "pull": 0, "push": 86911, "push_out_bytes": 34318722925, "recovery_ops": 86922, "recovery_bytes": 34314397051, "loadavg": 62, "cached_crc": 0, "cached_crc_adjusted": 0, "missed_crc": 0, "numpg": 81, "numpg_primary": 32, "numpg_replica": 49, "numpg_stray": 0, "numpg_removing": 0, "heartbeat_to_peers": 53, "map_messages": 3924, "map_message_epochs": 4465, "map_message_epoch_dups": 2957, "messages_delayed_for_map": 0, "osd_map_cache_hit": 127507, "osd_map_cache_miss": 22, "osd_map_cache_miss_low": 2, "osd_map_cache_miss_low_avg": { "avgcount": 2, "sum": 21 }, "osd_map_bl_cache_hit": 4254, "osd_map_bl_cache_miss": 2798, "stat_bytes": 10128816332800, "stat_bytes_used": 422762119168, "stat_bytes_avail": 9706054213632, "copyfrom": 0, "tier_promote": 0, "tier_flush": 0, "tier_flush_fail": 0, "tier_try_flush": 0, "tier_try_flush_fail": 0, "tier_evict": 0, "tier_whiteout": 0, "tier_dirty": 29567, "tier_clean": 0, "tier_delay": 0, "tier_proxy_read": 0, "tier_proxy_write": 0, "agent_wake": 0, "agent_skip": 0, "agent_flush": 0, "agent_evict": 0, "object_ctx_cache_hit": 288613, "object_ctx_cache_total": 488855, "op_cache_hit": 0, "osd_tier_flush_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_tier_promote_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_tier_r_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "osd_pg_info": 238380, "osd_pg_fastinfo": 207321, "osd_pg_biginfo": 1634 }, "recoverystate_perf": { "initial_latency": { "avgcount": 83, "sum": 1.762260823, "avgtime": 0.021232058 }, "started_latency": { "avgcount": 597, "sum": 12091840.383300855, "avgtime": 20254.339000503 }, "reset_latency": { "avgcount": 678, "sum": 105.680233720, "avgtime": 0.155870551 }, "start_latency": { "avgcount": 678, "sum": 0.013440001, "avgtime": 0.000019823 }, "primary_latency": { "avgcount": 229, "sum": 3916876.117159230, "avgtime": 17104.262520345 }, "peering_latency": { "avgcount": 261, "sum": 242.487618037, "avgtime": 0.929071333 }, "backfilling_latency": { "avgcount": 11, "sum": 28355.181184224, "avgtime": 2577.743744020 }, "waitremotebackfillreserved_latency": { "avgcount": 11, "sum": 0.010010103, "avgtime": 0.000910009 }, "waitlocalbackfillreserved_latency": { "avgcount": 11, "sum": 0.000914005, "avgtime": 0.000083091 }, "notbackfilling_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "repnotrecovering_latency": { "avgcount": 236, "sum": 8174666.478144644, "avgtime": 34638.417280273 }, "repwaitrecoveryreserved_latency": { "avgcount": 13, "sum": 0.000544600, "avgtime": 0.000041892 }, "repwaitbackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "reprecovering_latency": { "avgcount": 13, "sum": 5.236229915, "avgtime": 0.402786916 }, "activating_latency": { "avgcount": 226, "sum": 7.792077960, "avgtime": 0.034478221 }, "waitlocalrecoveryreserved_latency": { "avgcount": 6, "sum": 0.000532092, "avgtime": 0.000088682 }, "waitremoterecoveryreserved_latency": { "avgcount": 6, "sum": 0.005132390, "avgtime": 0.000855398 }, "recovering_latency": { "avgcount": 6, "sum": 2.188644385, "avgtime": 0.364774064 }, "recovered_latency": { "avgcount": 225, "sum": 0.004724875, "avgtime": 0.000020999 }, "clean_latency": { "avgcount": 193, "sum": 3888298.606566839, "avgtime": 20146.624904491 }, "active_latency": { "avgcount": 194, "sum": 3916662.571342664, "avgtime": 20188.982326508 }, "replicaactive_latency": { "avgcount": 223, "sum": 8174668.635187419, "avgtime": 36657.706884248 }, "stray_latency": { "avgcount": 417, "sum": 343.521093120, "avgtime": 0.823791590 }, "getinfo_latency": { "avgcount": 261, "sum": 26.466818426, "avgtime": 0.101405434 }, "getlog_latency": { "avgcount": 229, "sum": 0.313088774, "avgtime": 0.001367199 }, "waitactingchange_latency": { "avgcount": 3, "sum": 2.787477889, "avgtime": 0.929159296 }, "incomplete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "down_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "getmissing_latency": { "avgcount": 226, "sum": 0.070469958, "avgtime": 0.000311813 }, "waitupthru_latency": { "avgcount": 226, "sum": 215.630016592, "avgtime": 0.954115117 }, "notrecovering_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "rocksdb": { "get": 3900573, "get_latency": { "avgcount": 3900573, "sum": 46.497747616, "avgtime": 0.000011920 }, "submit_latency": { "avgcount": 241063, "sum": 14.648569398, "avgtime": 0.000060766 }, "submit_sync_latency": { "avgcount": 225099, "sum": 45.107860324, "avgtime": 0.000200391 }, "compact": 0, "compact_range": 0, "compact_queue_merge": 0, "compact_queue_len": 0, "rocksdb_write_wal_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_memtable_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_delay_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "rocksdb_write_pre_and_post_time": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-bluestore_throttle_bytes": { "val": 0, "max": 67108864, "get_started": 241063, "get": 241063, "get_sum": 1227078527883, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 225099, "put_sum": 1227078527883, "wait": { "avgcount": 2039, "sum": 65.096143721, "avgtime": 0.031925524 } }, "throttle-bluestore_throttle_deferred_bytes": { "val": 0, "max": 201326592, "get_started": 0, "get": 47230, "get_sum": 32111982926, "get_or_fail_fail": 0, "get_or_fail_success": 47230, "take": 0, "take_sum": 0, "put": 44540, "put_sum": 32111982926, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-client": { "val": 0, "max": 104857600, "get_started": 0, "get": 292633, "get_sum": 39290356304, "get_or_fail_fail": 0, "get_or_fail_success": 292633, "take": 0, "take_sum": 0, "put": 292633, "put_sum": 39290356304, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-cluster": { "val": 0, "max": 104857600, "get_started": 0, "get": 7182670, "get_sum": 60512426404, "get_or_fail_fail": 0, "get_or_fail_success": 7182670, "take": 0, "take_sum": 0, "put": 7182670, "put_sum": 60512426404, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_client": { "val": 0, "max": 104857600, "get_started": 0, "get": 7382217, "get_sum": 15008047161, "get_or_fail_fail": 0, "get_or_fail_success": 7382217, "take": 0, "take_sum": 0, "put": 7382217, "put_sum": 15008047161, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0, "max": 104857600, "get_started": 0, "get": 6979126, "get_sum": 14188562814, "get_or_fail_fail": 0, "get_or_fail_success": 6979126, "take": 0, "take_sum": 0, "put": 6979126, "put_sum": 14188562814, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_client": { "val": 0, "max": 104857600, "get_started": 0, "get": 7382217, "get_sum": 15008047161, "get_or_fail_fail": 0, "get_or_fail_success": 7382217, "take": 0, "take_sum": 0, "put": 7382217, "put_sum": 15008047161, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0, "max": 104857600, "get_started": 0, "get": 6979126, "get_sum": 14188562814, "get_or_fail_fail": 0, "get_or_fail_success": 6979126, "take": 0, "take_sum": 0, "put": 6979126, "put_sum": 14188562814, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 1024, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-osd_client_bytes": { "val": 47, "max": 524288000, "get_started": 0, "get": 291042, "get_sum": 39289027913, "get_or_fail_fail": 0, "get_or_fail_success": 291042, "take": 0, "take_sum": 0, "put": 1164143, "put_sum": 39289027866, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-osd_client_messages": { "val": 1, "max": 256, "get_started": 0, "get": 291042, "get_sum": 291042, "get_or_fail_fail": 0, "get_or_fail_success": 291042, "take": 0, "take_sum": 0, "put": 291041, "put_sum": 291041, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } } } and dump_mempools: { "mempool": { "by_pool": { "bloom_filter": { "items": 0, "bytes": 0 }, "bluestore_alloc": { "items": 155779, "bytes": 12462320 }, "bluestore_cache_data": { "items": 228476, "bytes": 233092536 }, "bluestore_cache_onode": { "items": 265318, "bytes": 163435888 }, "bluestore_cache_meta": { "items": 83890049, "bytes": 455300708 }, "bluestore_cache_other": { "items": 11355469, "bytes": 91930988 }, "bluestore_Buffer": { "items": 5325, "bytes": 511200 }, "bluestore_Extent": { "items": 842524, "bytes": 40441152 }, "bluestore_Blob": { "items": 842524, "bytes": 94362688 }, "bluestore_SharedBlob": { "items": 842524, "bytes": 94362688 }, "bluestore_inline_bl": { "items": 8842, "bytes": 1142714 }, "bluestore_fsck": { "items": 0, "bytes": 0 }, "bluestore_txc": { "items": 0, "bytes": 0 }, "bluestore_writing_deferred": { "items": 77, "bytes": 1798719 }, "bluestore_writing": { "items": 0, "bytes": 0 }, "bluefs": { "items": 1443, "bytes": 36800 }, "bluefs_file_reader": { "items": 96, "bytes": 2104832 }, "bluefs_file_writer": { "items": 3, "bytes": 576 }, "buffer_anon": { "items": 25898, "bytes": 10464534 }, "buffer_meta": { "items": 28808, "bytes": 2535104 }, "osd": { "items": 81, "bytes": 916272 }, "osd_mapbl": { "items": 0, "bytes": 0 }, "osd_pglog": { "items": 363769, "bytes": 164806408 }, "osdmap": { "items": 53839, "bytes": 2182392 }, "osdmap_mapping": { "items": 0, "bytes": 0 }, "pgmap": { "items": 0, "bytes": 0 }, "mds_co": { "items": 0, "bytes": 0 }, "unittest_1": { "items": 0, "bytes": 0 }, "unittest_2": { "items": 0, "bytes": 0 } }, "total": { "items": 98910844, "bytes": 1371888519 } } } Luis Domingues Proton AG ------- Original Message ------- On Tuesday, July 11th, 2023 at 14:59, Mark Nelson <mark.nelson@xxxxxxxxx> wrote: > Hi Luis, > > > Can you do a "ceph tell osd.<num> perf dump" and "ceph daemon osd.<num> > > dump_mempools"? Those should help us understand how much memory is > being used by different parts of the OSD/bluestore and how much memory > the priority cache thinks it has to work with. > > > Mark > > On 7/11/23 4:57 AM, Luis Domingues wrote: > > > Hi everyone, > > > > We recently migrate a cluster from ceph-ansible to cephadm. Everything went as expected. > > But now we have some alerts on high memory usage. Cluster is running ceph 16.2.13. > > > > Of course, after adoption OSDs ended up in the <unmanaged> zone: > > > > NAME PORTS RUNNING REFRESHED AGE PLACEMENT > > osd 88 7m ago - <unmanaged> > > > > But the weirdest thing I observed, is that the OSDs seem to use more memory that the mem limit: > > > > NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID > > osd.0 <node> running (5d) 2m ago 5d 19.7G 6400M 16.2.13 327f301eff51 ca07fe74a0fa > > osd.1 <node> running (5d) 2m ago 5d 7068M 6400M 16.2.13 327f301eff51 6223ed8e34e9 > > osd.10 <node> running (5d) 10m ago 5d 7235M 6400M 16.2.13 327f301eff51 073ddc0d7391 osd.100 <node> running (5d) 2m ago 5d 7118M 6400M 16.2.13 327f301eff51 b7f9238c0c24 > > > > Does anybody knows why OSDs would use more memory than the limit? > > > > Thanks > > > > Luis Domingues > > Proton AG > > _______________________________________________ > > ceph-users mailing list -- ceph-users@xxxxxxx > > To unsubscribe send an email to ceph-users-leave@xxxxxxx > > > -- > Best Regards, > Mark Nelson > Head of R&D (USA) > > Clyso GmbH > p: +49 89 21552391 12 > a: Loristraße 8 | 80335 München | Germany > w: https://clyso.com | e: mark.nelson@xxxxxxxxx > > We are hiring: https://www.clyso.com/jobs/ > _______________________________________________ > ceph-users mailing list -- ceph-users@xxxxxxx > To unsubscribe send an email to ceph-users-leave@xxxxxxx _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx