Re: OSD memory usage after cephadm adoption

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Here you have. Perf dump:

{
    "AsyncMessenger::Worker-0": {
        "msgr_recv_messages": 12239872,
        "msgr_send_messages": 12284221,
        "msgr_recv_bytes": 43759275160,
        "msgr_send_bytes": 61268769426,
        "msgr_created_connections": 754,
        "msgr_active_connections": 100,
        "msgr_running_total_time": 939.476931816,
        "msgr_running_send_time": 337.873686715,
        "msgr_running_recv_time": 360.728238752,
        "msgr_running_fast_dispatch_time": 183.737116872,
        "msgr_send_messages_queue_lat": {
            "avgcount": 12284206,
            "sum": 1538.989479364,
            "avgtime": 0.000125281
        },
        "msgr_handle_ack_lat": {
            "avgcount": 5258403,
            "sum": 1.005075918,
            "avgtime": 0.000000191
        }
    },
    "AsyncMessenger::Worker-1": {
        "msgr_recv_messages": 12099771,
        "msgr_send_messages": 12138795,
        "msgr_recv_bytes": 56967534605,
        "msgr_send_bytes": 130548664272,
        "msgr_created_connections": 647,
        "msgr_active_connections": 91,
        "msgr_running_total_time": 977.277996439,
        "msgr_running_send_time": 362.155959231,
        "msgr_running_recv_time": 365.376281473,
        "msgr_running_fast_dispatch_time": 191.186643292,
        "msgr_send_messages_queue_lat": {
            "avgcount": 12138818,
            "sum": 1557.187685700,
            "avgtime": 0.000128281
        },
        "msgr_handle_ack_lat": {
            "avgcount": 6155265,
            "sum": 1.096270527,
            "avgtime": 0.000000178
        }
    },
    "AsyncMessenger::Worker-2": {
        "msgr_recv_messages": 11858354,
        "msgr_send_messages": 11960404,
        "msgr_recv_bytes": 60727084610,
        "msgr_send_bytes": 168534726650,
        "msgr_created_connections": 1043,
        "msgr_active_connections": 103,
        "msgr_running_total_time": 937.324084772,
        "msgr_running_send_time": 351.174710644,
        "msgr_running_recv_time": 2744.276782474,
        "msgr_running_fast_dispatch_time": 172.960322050,
        "msgr_send_messages_queue_lat": {
            "avgcount": 11960392,
            "sum": 1763.762581924,
            "avgtime": 0.000147466
        },
        "msgr_handle_ack_lat": {
            "avgcount": 2651457,
            "sum": 0.538495450,
            "avgtime": 0.000000203
        }
    },
    "bluefs": {
        "db_total_bytes": 128005955584,
        "db_used_bytes": 3271557120,
        "wal_total_bytes": 0,
        "wal_used_bytes": 0,
        "slow_total_bytes": 10000810377216,
        "slow_used_bytes": 0,
        "num_files": 70,
        "log_bytes": 13045760,
        "log_compactions": 58,
        "logged_bytes": 922333184,
        "files_written_wal": 2,
        "files_written_sst": 13,
        "bytes_written_wal": 1988489216,
        "bytes_written_sst": 268890112,
        "bytes_written_slow": 0,
        "max_bytes_wal": 0,
        "max_bytes_db": 3271557120,
        "max_bytes_slow": 0,
        "read_random_count": 577484,
        "read_random_bytes": 2879541532,
        "read_random_disk_count": 284290,
        "read_random_disk_bytes": 1540394118,
        "read_random_buffer_count": 319088,
        "read_random_buffer_bytes": 1339147414,
        "read_count": 1086625,
        "read_bytes": 15054317429,
        "read_prefetch_count": 1069462,
        "read_prefetch_bytes": 14506469332,
        "read_zeros_candidate": 0,
        "read_zeros_errors": 0
    },
    "bluestore": {
        "kv_flush_lat": {
            "avgcount": 225099,
            "sum": 526.605165277,
            "avgtime": 0.002339438
        },
        "kv_commit_lat": {
            "avgcount": 225099,
            "sum": 61.412175620,
            "avgtime": 0.000272822
        },
        "kv_sync_lat": {
            "avgcount": 225099,
            "sum": 588.017340897,
            "avgtime": 0.002612261
        },
        "kv_final_lat": {
            "avgcount": 225096,
            "sum": 6.516869320,
            "avgtime": 0.000028951
        },
        "state_prepare_lat": {
            "avgcount": 241063,
            "sum": 173.705759592,
            "avgtime": 0.000720582
        },
        "state_aio_wait_lat": {
            "avgcount": 241063,
            "sum": 1008.936150524,
            "avgtime": 0.004185362
        },
        "state_io_done_lat": {
            "avgcount": 241063,
            "sum": 2.923457351,
            "avgtime": 0.000012127
        },
        "state_kv_queued_lat": {
            "avgcount": 241063,
            "sum": 560.050193021,
            "avgtime": 0.002323252
        },
        "state_kv_commiting_lat": {
            "avgcount": 241063,
            "sum": 68.355225981,
            "avgtime": 0.000283557
        },
        "state_kv_done_lat": {
            "avgcount": 241063,
            "sum": 0.097836444,
            "avgtime": 0.000000405
        },
        "state_deferred_queued_lat": {
            "avgcount": 47230,
            "sum": 70711.960011607,
            "avgtime": 1.497183146
        },
        "state_deferred_aio_wait_lat": {
            "avgcount": 47230,
            "sum": 70.931594687,
            "avgtime": 0.001501833
        },
        "state_deferred_cleanup_lat": {
            "avgcount": 47229,
            "sum": 333702.384485631,
            "avgtime": 7.065624605
        },
        "state_finishing_lat": {
            "avgcount": 241062,
            "sum": 0.033475853,
            "avgtime": 0.000000138
        },
        "state_done_lat": {
            "avgcount": 241062,
            "sum": 15258.375631838,
            "avgtime": 0.063296478
        },
        "throttle_lat": {
            "avgcount": 241063,
            "sum": 67.464167288,
            "avgtime": 0.000279861
        },
        "submit_lat": {
            "avgcount": 241063,
            "sum": 190.468701838,
            "avgtime": 0.000790120
        },
        "commit_lat": {
            "avgcount": 241063,
            "sum": 1814.026687331,
            "avgtime": 0.007525114
        },
        "read_lat": {
            "avgcount": 1590088,
            "sum": 8928.843945649,
            "avgtime": 0.005615314
        },
        "read_onode_meta_lat": {
            "avgcount": 3106127,
            "sum": 18.873640476,
            "avgtime": 0.000006076
        },
        "read_wait_aio_lat": {
            "avgcount": 1516039,
            "sum": 8830.114048334,
            "avgtime": 0.005824463
        },
        "compress_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "decompress_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "csum_lat": {
            "avgcount": 7047778,
            "sum": 52.140862362,
            "avgtime": 0.000007398
        },
        "compress_success_count": 0,
        "compress_rejected_count": 0,
        "write_pad_bytes": 194870362,
        "deferred_write_ops": 50669,
        "deferred_write_bytes": 388481024,
        "write_penalty_read_ops": 2067,
        "bluestore_allocated": 294756155392,
        "bluestore_stored": 292495538258,
        "bluestore_compressed": 0,
        "bluestore_compressed_allocated": 0,
        "bluestore_compressed_original": 0,
        "bluestore_onodes": 266002,
        "bluestore_pinned_onodes": 1,
        "onode_hits": 8782379,
        "onode_misses": 6047682,
        "onode_shard_hits": 1277688,
        "bluestore_onode_shard_misses": 947296,
        "bluestore_extents": 840549,
        "bluestore_blobs": 840549,
        "bluestore_buffers": 3591,
        "bluestore_buffer_bytes": 224182272,
        "bluestore_buffer_hit_bytes": 110213431882,
        "bluestore_buffer_miss_bytes": 406742926152,
        "bluestore_write_big": 39721,
        "bluestore_write_big_bytes": 94625427456,
        "bluestore_write_big_blobs": 1450312,
        "bluestore_write_big_deferred": 0,
        "bluestore_write_small": 60521,
        "bluestore_write_small_bytes": 48842873,
        "bluestore_write_small_unused": 926,
        "bluestore_write_deferred": 62388,
        "bluestore_write_deferred_bytes": 398618624,
        "bluestore_write_small_pre_read": 7434,
        "bluestore_write_new": 1448445,
        "bluestore_txc": 241063,
        "bluestore_onode_reshard": 27825,
        "bluestore_blob_split": 5,
        "bluestore_extent_compress": 3758,
        "bluestore_gc_merged": 0,
        "bluestore_read_eio": 0,
        "bluestore_reads_with_retries": 0,
        "bluestore_fragmentation_micros": 0,
        "omap_seek_to_first_lat": {
            "avgcount": 759176,
            "sum": 8.351645295,
            "avgtime": 0.000011000
        },
        "omap_upper_bound_lat": {
            "avgcount": 308,
            "sum": 0.002216099,
            "avgtime": 0.000007195
        },
        "omap_lower_bound_lat": {
            "avgcount": 87154,
            "sum": 1.192486653,
            "avgtime": 0.000013682
        },
        "omap_next_lat": {
            "avgcount": 308598,
            "sum": 0.211502859,
            "avgtime": 0.000000685
        },
        "omap_get_keys_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "omap_get_values_lat": {
            "avgcount": 19348,
            "sum": 0.032918565,
            "avgtime": 0.000001701
        },
        "clist_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "remove_lat": {
            "avgcount": 239512,
            "sum": 121.690074965,
            "avgtime": 0.000508075
        }
    },
    "bluestore-pricache": {
        "target_bytes": 6713193267,
        "mapped_bytes": 6718742528,
        "unmapped_bytes": 467025920,
        "heap_bytes": 7185768448,
        "cache_bytes": 4161537138
    },
    "bluestore-pricache:data": {
        "pri0_bytes": 0,
        "pri1_bytes": 224182272,
        "pri2_bytes": 0,
        "pri3_bytes": 0,
        "pri4_bytes": 0,
        "pri5_bytes": 0,
        "pri6_bytes": 0,
        "pri7_bytes": 0,
        "pri8_bytes": 0,
        "pri9_bytes": 0,
        "pri10_bytes": 0,
        "pri11_bytes": 30742099,
        "reserved_bytes": 80619949,
        "committed_bytes": 335544320
    },
    "bluestore-pricache:kv": {
        "pri0_bytes": 0,
        "pri1_bytes": 176540224,
        "pri2_bytes": 0,
        "pri3_bytes": 0,
        "pri4_bytes": 0,
        "pri5_bytes": 0,
        "pri6_bytes": 0,
        "pri7_bytes": 0,
        "pri8_bytes": 0,
        "pri9_bytes": 0,
        "pri10_bytes": 0,
        "pri11_bytes": 230565745,
        "reserved_bytes": 79433295,
        "committed_bytes": 486539264
    },
    "bluestore-pricache:kv_onode": {
        "pri0_bytes": 0,
        "pri1_bytes": 1977148816,
        "pri2_bytes": 0,
        "pri3_bytes": 0,
        "pri4_bytes": 0,
        "pri5_bytes": 0,
        "pri6_bytes": 0,
        "pri7_bytes": 0,
        "pri8_bytes": 0,
        "pri9_bytes": 0,
        "pri10_bytes": 0,
        "pri11_bytes": 20494732,
        "reserved_bytes": 82731236,
        "committed_bytes": 2080374784
    },
    "bluestore-pricache:meta": {
        "pri0_bytes": 0,
        "pri1_bytes": 942217320,
        "pri2_bytes": 0,
        "pri3_bytes": 0,
        "pri4_bytes": 0,
        "pri5_bytes": 0,
        "pri6_bytes": 0,
        "pri7_bytes": 0,
        "pri8_bytes": 0,
        "pri9_bytes": 0,
        "pri10_bytes": 0,
        "pri11_bytes": 230565745,
        "reserved_bytes": 68730919,
        "committed_bytes": 1241513984
    },
    "cct": {
        "total_workers": 5,
        "unhealthy_workers": 0
    },
    "finisher-commit_finisher": {
        "queue_len": 0,
        "complete_latency": {
            "avgcount": 1507,
            "sum": 0.568110372,
            "avgtime": 0.000376981
        }
    },
    "finisher-objecter-finisher-0": {
        "queue_len": 0,
        "complete_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "mempool": {
        "bloom_filter_bytes": 0,
        "bloom_filter_items": 0,
        "bluestore_alloc_bytes": 12461120,
        "bluestore_alloc_items": 155764,
        "bluestore_cache_data_bytes": 233998707,
        "bluestore_cache_data_items": 229226,
        "bluestore_cache_onode_bytes": 163857232,
        "bluestore_cache_onode_items": 266002,
        "bluestore_cache_meta_bytes": 456521298,
        "bluestore_cache_meta_items": 84112717,
        "bluestore_cache_other_bytes": 91560856,
        "bluestore_cache_other_items": 11308804,
        "bluestore_Buffer_bytes": 513888,
        "bluestore_Buffer_items": 5353,
        "bluestore_Extent_bytes": 40346352,
        "bluestore_Extent_items": 840549,
        "bluestore_Blob_bytes": 94141488,
        "bluestore_Blob_items": 840549,
        "bluestore_SharedBlob_bytes": 94141488,
        "bluestore_SharedBlob_items": 840549,
        "bluestore_inline_bl_bytes": 1134718,
        "bluestore_inline_bl_items": 8758,
        "bluestore_fsck_bytes": 0,
        "bluestore_fsck_items": 0,
        "bluestore_txc_bytes": 784,
        "bluestore_txc_items": 1,
        "bluestore_writing_deferred_bytes": 1918388,
        "bluestore_writing_deferred_items": 81,
        "bluestore_writing_bytes": 0,
        "bluestore_writing_items": 0,
        "bluefs_bytes": 36800,
        "bluefs_items": 1443,
        "bluefs_file_reader_bytes": 1842688,
        "bluefs_file_reader_items": 95,
        "bluefs_file_writer_bytes": 576,
        "bluefs_file_writer_items": 3,
        "buffer_anon_bytes": 10746933,
        "buffer_anon_items": 25967,
        "buffer_meta_bytes": 2541352,
        "buffer_meta_items": 28879,
        "osd_bytes": 916272,
        "osd_items": 81,
        "osd_mapbl_bytes": 0,
        "osd_mapbl_items": 0,
        "osd_pglog_bytes": 164733192,
        "osd_pglog_items": 363565,
        "osdmap_bytes": 2182392,
        "osdmap_items": 53839,
        "osdmap_mapping_bytes": 0,
        "osdmap_mapping_items": 0,
        "pgmap_bytes": 0,
        "pgmap_items": 0,
        "mds_co_bytes": 0,
        "mds_co_items": 0,
        "unittest_1_bytes": 0,
        "unittest_1_items": 0,
        "unittest_2_bytes": 0,
        "unittest_2_items": 0
    },
    "objecter": {
        "op_active": 0,
        "op_laggy": 0,
        "op_send": 0,
        "op_send_bytes": 0,
        "op_resend": 0,
        "op_reply": 0,
        "oplen_avg": {
            "avgcount": 0,
            "sum": 0
        },
        "op": 0,
        "op_r": 0,
        "op_w": 0,
        "op_rmw": 0,
        "op_pg": 0,
        "osdop_stat": 0,
        "osdop_create": 0,
        "osdop_read": 0,
        "osdop_write": 0,
        "osdop_writefull": 0,
        "osdop_writesame": 0,
        "osdop_append": 0,
        "osdop_zero": 0,
        "osdop_truncate": 0,
        "osdop_delete": 0,
        "osdop_mapext": 0,
        "osdop_sparse_read": 0,
        "osdop_clonerange": 0,
        "osdop_getxattr": 0,
        "osdop_setxattr": 0,
        "osdop_cmpxattr": 0,
        "osdop_rmxattr": 0,
        "osdop_resetxattrs": 0,
        "osdop_call": 0,
        "osdop_watch": 0,
        "osdop_notify": 0,
        "osdop_src_cmpxattr": 0,
        "osdop_pgls": 0,
        "osdop_pgls_filter": 0,
        "osdop_other": 0,
        "linger_active": 0,
        "linger_send": 0,
        "linger_resend": 0,
        "linger_ping": 0,
        "poolop_active": 0,
        "poolop_send": 0,
        "poolop_resend": 0,
        "poolstat_active": 0,
        "poolstat_send": 0,
        "poolstat_resend": 0,
        "statfs_active": 0,
        "statfs_send": 0,
        "statfs_resend": 0,
        "command_active": 0,
        "command_send": 0,
        "command_resend": 0,
        "map_epoch": 63038,
        "map_full": 2,
        "map_inc": 1511,
        "osd_sessions": 0,
        "osd_session_open": 0,
        "osd_session_close": 0,
        "osd_laggy": 0,
        "omap_wr": 0,
        "omap_rd": 0,
        "omap_del": 0
    },
    "osd": {
        "op_wip": 0,
        "op": 272274,
        "op_in_bytes": 39148504708,
        "op_out_bytes": 181035613199,
        "op_latency": {
            "avgcount": 272274,
            "sum": 2312.558437554,
            "avgtime": 0.008493497
        },
        "op_process_latency": {
            "avgcount": 272274,
            "sum": 2151.138778914,
            "avgtime": 0.007900639
        },
        "op_prepare_latency": {
            "avgcount": 280169,
            "sum": 1068.077633101,
            "avgtime": 0.003812262
        },
        "op_r": 191667,
        "op_r_out_bytes": 181035613199,
        "op_r_latency": {
            "avgcount": 191667,
            "sum": 1035.691613777,
            "avgtime": 0.005403599
        },
        "op_r_process_latency": {
            "avgcount": 191667,
            "sum": 959.303890513,
            "avgtime": 0.005005055
        },
        "op_r_prepare_latency": {
            "avgcount": 191712,
            "sum": 962.370649266,
            "avgtime": 0.005019876
        },
        "op_w": 34260,
        "op_w_in_bytes": 39121651122,
        "op_w_latency": {
            "avgcount": 34260,
            "sum": 1117.333684176,
            "avgtime": 0.032613359
        },
        "op_w_process_latency": {
            "avgcount": 34260,
            "sum": 1059.202697781,
            "avgtime": 0.030916599
        },
        "op_w_prepare_latency": {
            "avgcount": 36798,
            "sum": 91.420473715,
            "avgtime": 0.002484387
        },
        "op_rw": 46347,
        "op_rw_in_bytes": 26853586,
        "op_rw_out_bytes": 0,
        "op_rw_latency": {
            "avgcount": 46347,
            "sum": 159.533139601,
            "avgtime": 0.003442145
        },
        "op_rw_process_latency": {
            "avgcount": 46347,
            "sum": 132.632190620,
            "avgtime": 0.002861721
        },
        "op_rw_prepare_latency": {
            "avgcount": 51659,
            "sum": 14.286510120,
            "avgtime": 0.000276554
        },
        "op_before_queue_op_lat": {
            "avgcount": 897996,
            "sum": 154.309110719,
            "avgtime": 0.000171837
        },
        "op_before_dequeue_op_lat": {
            "avgcount": 901696,
            "sum": 622.443106332,
            "avgtime": 0.000690302
        },
        "subop": 119722,
        "subop_in_bytes": 55578084121,
        "subop_latency": {
            "avgcount": 119722,
            "sum": 1159.173264760,
            "avgtime": 0.009682207
        },
        "subop_w": 119722,
        "subop_w_in_bytes": 55578084121,
        "subop_w_latency": {
            "avgcount": 119722,
            "sum": 1159.173264760,
            "avgtime": 0.009682207
        },
        "subop_pull": 0,
        "subop_pull_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "subop_push": 0,
        "subop_push_in_bytes": 0,
        "subop_push_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "pull": 0,
        "push": 86911,
        "push_out_bytes": 34318722925,
        "recovery_ops": 86922,
        "recovery_bytes": 34314397051,
        "loadavg": 62,
        "cached_crc": 0,
        "cached_crc_adjusted": 0,
        "missed_crc": 0,
        "numpg": 81,
        "numpg_primary": 32,
        "numpg_replica": 49,
        "numpg_stray": 0,
        "numpg_removing": 0,
        "heartbeat_to_peers": 53,
        "map_messages": 3924,
        "map_message_epochs": 4465,
        "map_message_epoch_dups": 2957,
        "messages_delayed_for_map": 0,
        "osd_map_cache_hit": 127507,
        "osd_map_cache_miss": 22,
        "osd_map_cache_miss_low": 2,
        "osd_map_cache_miss_low_avg": {
            "avgcount": 2,
            "sum": 21
        },
        "osd_map_bl_cache_hit": 4254,
        "osd_map_bl_cache_miss": 2798,
        "stat_bytes": 10128816332800,
        "stat_bytes_used": 422762119168,
        "stat_bytes_avail": 9706054213632,
        "copyfrom": 0,
        "tier_promote": 0,
        "tier_flush": 0,
        "tier_flush_fail": 0,
        "tier_try_flush": 0,
        "tier_try_flush_fail": 0,
        "tier_evict": 0,
        "tier_whiteout": 0,
        "tier_dirty": 29567,
        "tier_clean": 0,
        "tier_delay": 0,
        "tier_proxy_read": 0,
        "tier_proxy_write": 0,
        "agent_wake": 0,
        "agent_skip": 0,
        "agent_flush": 0,
        "agent_evict": 0,
        "object_ctx_cache_hit": 288613,
        "object_ctx_cache_total": 488855,
        "op_cache_hit": 0,
        "osd_tier_flush_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "osd_tier_promote_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "osd_tier_r_lat": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "osd_pg_info": 238380,
        "osd_pg_fastinfo": 207321,
        "osd_pg_biginfo": 1634
    },
    "recoverystate_perf": {
        "initial_latency": {
            "avgcount": 83,
            "sum": 1.762260823,
            "avgtime": 0.021232058
        },
        "started_latency": {
            "avgcount": 597,
            "sum": 12091840.383300855,
            "avgtime": 20254.339000503
        },
        "reset_latency": {
            "avgcount": 678,
            "sum": 105.680233720,
            "avgtime": 0.155870551
        },
        "start_latency": {
            "avgcount": 678,
            "sum": 0.013440001,
            "avgtime": 0.000019823
        },
        "primary_latency": {
            "avgcount": 229,
            "sum": 3916876.117159230,
            "avgtime": 17104.262520345
        },
        "peering_latency": {
            "avgcount": 261,
            "sum": 242.487618037,
            "avgtime": 0.929071333
        },
        "backfilling_latency": {
            "avgcount": 11,
            "sum": 28355.181184224,
            "avgtime": 2577.743744020
        },
        "waitremotebackfillreserved_latency": {
            "avgcount": 11,
            "sum": 0.010010103,
            "avgtime": 0.000910009
        },
        "waitlocalbackfillreserved_latency": {
            "avgcount": 11,
            "sum": 0.000914005,
            "avgtime": 0.000083091
        },
        "notbackfilling_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "repnotrecovering_latency": {
            "avgcount": 236,
            "sum": 8174666.478144644,
            "avgtime": 34638.417280273
        },
        "repwaitrecoveryreserved_latency": {
            "avgcount": 13,
            "sum": 0.000544600,
            "avgtime": 0.000041892
        },
        "repwaitbackfillreserved_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "reprecovering_latency": {
            "avgcount": 13,
            "sum": 5.236229915,
            "avgtime": 0.402786916
        },
        "activating_latency": {
            "avgcount": 226,
            "sum": 7.792077960,
            "avgtime": 0.034478221
        },
        "waitlocalrecoveryreserved_latency": {
            "avgcount": 6,
            "sum": 0.000532092,
            "avgtime": 0.000088682
        },
        "waitremoterecoveryreserved_latency": {
            "avgcount": 6,
            "sum": 0.005132390,
            "avgtime": 0.000855398
        },
        "recovering_latency": {
            "avgcount": 6,
            "sum": 2.188644385,
            "avgtime": 0.364774064
        },
        "recovered_latency": {
            "avgcount": 225,
            "sum": 0.004724875,
            "avgtime": 0.000020999
        },
        "clean_latency": {
            "avgcount": 193,
            "sum": 3888298.606566839,
            "avgtime": 20146.624904491
        },
        "active_latency": {
            "avgcount": 194,
            "sum": 3916662.571342664,
            "avgtime": 20188.982326508
        },
        "replicaactive_latency": {
            "avgcount": 223,
            "sum": 8174668.635187419,
            "avgtime": 36657.706884248
        },
        "stray_latency": {
            "avgcount": 417,
            "sum": 343.521093120,
            "avgtime": 0.823791590
        },
        "getinfo_latency": {
            "avgcount": 261,
            "sum": 26.466818426,
            "avgtime": 0.101405434
        },
        "getlog_latency": {
            "avgcount": 229,
            "sum": 0.313088774,
            "avgtime": 0.001367199
        },
        "waitactingchange_latency": {
            "avgcount": 3,
            "sum": 2.787477889,
            "avgtime": 0.929159296
        },
        "incomplete_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "down_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "getmissing_latency": {
            "avgcount": 226,
            "sum": 0.070469958,
            "avgtime": 0.000311813
        },
        "waitupthru_latency": {
            "avgcount": 226,
            "sum": 215.630016592,
            "avgtime": 0.954115117
        },
        "notrecovering_latency": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "rocksdb": {
        "get": 3900573,
        "get_latency": {
            "avgcount": 3900573,
            "sum": 46.497747616,
            "avgtime": 0.000011920
        },
        "submit_latency": {
            "avgcount": 241063,
            "sum": 14.648569398,
            "avgtime": 0.000060766
        },
        "submit_sync_latency": {
            "avgcount": 225099,
            "sum": 45.107860324,
            "avgtime": 0.000200391
        },
        "compact": 0,
        "compact_range": 0,
        "compact_queue_merge": 0,
        "compact_queue_len": 0,
        "rocksdb_write_wal_time": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "rocksdb_write_memtable_time": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "rocksdb_write_delay_time": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        },
        "rocksdb_write_pre_and_post_time": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-bluestore_throttle_bytes": {
        "val": 0,
        "max": 67108864,
        "get_started": 241063,
        "get": 241063,
        "get_sum": 1227078527883,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 0,
        "take": 0,
        "take_sum": 0,
        "put": 225099,
        "put_sum": 1227078527883,
        "wait": {
            "avgcount": 2039,
            "sum": 65.096143721,
            "avgtime": 0.031925524
        }
    },
    "throttle-bluestore_throttle_deferred_bytes": {
        "val": 0,
        "max": 201326592,
        "get_started": 0,
        "get": 47230,
        "get_sum": 32111982926,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 47230,
        "take": 0,
        "take_sum": 0,
        "put": 44540,
        "put_sum": 32111982926,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-client": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 292633,
        "get_sum": 39290356304,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 292633,
        "take": 0,
        "take_sum": 0,
        "put": 292633,
        "put_sum": 39290356304,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-cluster": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 7182670,
        "get_sum": 60512426404,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 7182670,
        "take": 0,
        "take_sum": 0,
        "put": 7182670,
        "put_sum": 60512426404,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-hb_back_client": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 7382217,
        "get_sum": 15008047161,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 7382217,
        "take": 0,
        "take_sum": 0,
        "put": 7382217,
        "put_sum": 15008047161,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-hb_back_server": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 6979126,
        "get_sum": 14188562814,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 6979126,
        "take": 0,
        "take_sum": 0,
        "put": 6979126,
        "put_sum": 14188562814,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-hb_front_client": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 7382217,
        "get_sum": 15008047161,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 7382217,
        "take": 0,
        "take_sum": 0,
        "put": 7382217,
        "put_sum": 15008047161,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-hb_front_server": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 6979126,
        "get_sum": 14188562814,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 6979126,
        "take": 0,
        "take_sum": 0,
        "put": 6979126,
        "put_sum": 14188562814,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-msgr_dispatch_throttler-ms_objecter": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 0,
        "get_sum": 0,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 0,
        "take": 0,
        "take_sum": 0,
        "put": 0,
        "put_sum": 0,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-objecter_bytes": {
        "val": 0,
        "max": 104857600,
        "get_started": 0,
        "get": 0,
        "get_sum": 0,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 0,
        "take": 0,
        "take_sum": 0,
        "put": 0,
        "put_sum": 0,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-objecter_ops": {
        "val": 0,
        "max": 1024,
        "get_started": 0,
        "get": 0,
        "get_sum": 0,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 0,
        "take": 0,
        "take_sum": 0,
        "put": 0,
        "put_sum": 0,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-osd_client_bytes": {
        "val": 47,
        "max": 524288000,
        "get_started": 0,
        "get": 291042,
        "get_sum": 39289027913,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 291042,
        "take": 0,
        "take_sum": 0,
        "put": 1164143,
        "put_sum": 39289027866,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    },
    "throttle-osd_client_messages": {
        "val": 1,
        "max": 256,
        "get_started": 0,
        "get": 291042,
        "get_sum": 291042,
        "get_or_fail_fail": 0,
        "get_or_fail_success": 291042,
        "take": 0,
        "take_sum": 0,
        "put": 291041,
        "put_sum": 291041,
        "wait": {
            "avgcount": 0,
            "sum": 0.000000000,
            "avgtime": 0.000000000
        }
    }
}

and dump_mempools:

{
    "mempool": {
        "by_pool": {
            "bloom_filter": {
                "items": 0,
                "bytes": 0
            },
            "bluestore_alloc": {
                "items": 155779,
                "bytes": 12462320
            },
            "bluestore_cache_data": {
                "items": 228476,
                "bytes": 233092536
            },
            "bluestore_cache_onode": {
                "items": 265318,
                "bytes": 163435888
            },
            "bluestore_cache_meta": {
                "items": 83890049,
                "bytes": 455300708
            },
            "bluestore_cache_other": {
                "items": 11355469,
                "bytes": 91930988
            },
            "bluestore_Buffer": {
                "items": 5325,
                "bytes": 511200
            },
            "bluestore_Extent": {
                "items": 842524,
                "bytes": 40441152
            },
            "bluestore_Blob": {
                "items": 842524,
                "bytes": 94362688
            },
            "bluestore_SharedBlob": {
                "items": 842524,
                "bytes": 94362688
            },
            "bluestore_inline_bl": {
                "items": 8842,
                "bytes": 1142714
            },
            "bluestore_fsck": {
                "items": 0,
                "bytes": 0
            },
            "bluestore_txc": {
                "items": 0,
                "bytes": 0
            },
            "bluestore_writing_deferred": {
                "items": 77,
                "bytes": 1798719
            },
            "bluestore_writing": {
                "items": 0,
                "bytes": 0
            },
            "bluefs": {
                "items": 1443,
                "bytes": 36800
            },
            "bluefs_file_reader": {
                "items": 96,
                "bytes": 2104832
            },
            "bluefs_file_writer": {
                "items": 3,
                "bytes": 576
            },
            "buffer_anon": {
                "items": 25898,
                "bytes": 10464534
            },
            "buffer_meta": {
                "items": 28808,
                "bytes": 2535104
            },
            "osd": {
                "items": 81,
                "bytes": 916272
            },
            "osd_mapbl": {
                "items": 0,
                "bytes": 0
            },
            "osd_pglog": {
                "items": 363769,
                "bytes": 164806408
            },
            "osdmap": {
                "items": 53839,
                "bytes": 2182392
            },
            "osdmap_mapping": {
                "items": 0,
                "bytes": 0
            },
            "pgmap": {
                "items": 0,
                "bytes": 0
            },
            "mds_co": {
                "items": 0,
                "bytes": 0
            },
            "unittest_1": {
                "items": 0,
                "bytes": 0
            },
            "unittest_2": {
                "items": 0,
                "bytes": 0
            }
        },
        "total": {
            "items": 98910844,
            "bytes": 1371888519
        }
    }
}

Luis Domingues
Proton AG


------- Original Message -------
On Tuesday, July 11th, 2023 at 14:59, Mark Nelson <mark.nelson@xxxxxxxxx> wrote:


> Hi Luis,
> 
> 
> Can you do a "ceph tell osd.<num> perf dump" and "ceph daemon osd.<num>
> 
> dump_mempools"? Those should help us understand how much memory is
> being used by different parts of the OSD/bluestore and how much memory
> the priority cache thinks it has to work with.
> 
> 
> Mark
> 
> On 7/11/23 4:57 AM, Luis Domingues wrote:
> 
> > Hi everyone,
> > 
> > We recently migrate a cluster from ceph-ansible to cephadm. Everything went as expected.
> > But now we have some alerts on high memory usage. Cluster is running ceph 16.2.13.
> > 
> > Of course, after adoption OSDs ended up in the <unmanaged> zone:
> > 
> > NAME PORTS RUNNING REFRESHED AGE PLACEMENT
> > osd 88 7m ago - <unmanaged>
> > 
> > But the weirdest thing I observed, is that the OSDs seem to use more memory that the mem limit:
> > 
> > NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID
> > osd.0 <node> running (5d) 2m ago 5d 19.7G 6400M 16.2.13 327f301eff51 ca07fe74a0fa
> > osd.1 <node> running (5d) 2m ago 5d 7068M 6400M 16.2.13 327f301eff51 6223ed8e34e9
> > osd.10 <node> running (5d) 10m ago 5d 7235M 6400M 16.2.13 327f301eff51 073ddc0d7391 osd.100 <node> running (5d) 2m ago 5d 7118M 6400M 16.2.13 327f301eff51 b7f9238c0c24
> > 
> > Does anybody knows why OSDs would use more memory than the limit?
> > 
> > Thanks
> > 
> > Luis Domingues
> > Proton AG
> > _______________________________________________
> > ceph-users mailing list -- ceph-users@xxxxxxx
> > To unsubscribe send an email to ceph-users-leave@xxxxxxx
> 
> 
> --
> Best Regards,
> Mark Nelson
> Head of R&D (USA)
> 
> Clyso GmbH
> p: +49 89 21552391 12
> a: Loristraße 8 | 80335 München | Germany
> w: https://clyso.com | e: mark.nelson@xxxxxxxxx
> 
> We are hiring: https://www.clyso.com/jobs/
> _______________________________________________
> ceph-users mailing list -- ceph-users@xxxxxxx
> To unsubscribe send an email to ceph-users-leave@xxxxxxx
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx




[Index of Archives]     [Information on CEPH]     [Linux Filesystem Development]     [Ceph Development]     [Ceph Large]     [Ceph Dev]     [Linux USB Development]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]     [xfs]


  Powered by Linux