|
Hello all,
We are using Ceph as the storage backend for some Cloud research which involves offloading functions to storage nodes to benefit from near-storage processing. We are using rados_exec to achieve this by attempting to call a class method on the object which then
executes the function locally. However, we have been running into an issue where rados_exec fails with EIO and the request is never reaching the storage node with method never being called.
Upon debugging this, I have noticed that if i re-put the same object with a different key it works (provided it is on a different OSD). It appears that the OSD cannot serve a rados_exec request.
This bug happens under a few conditions
I cannot seem to debug it for the life of me and only thing I have to go on is the OSDs cannot serve requests. I have attempted to remove the object from the pool and put it back with the same key and it does the exact same thing.
Any advice on where to get started / help debugging this would be greatly appreciated as my thesis depends on it. (any request to OSD.0 and OSD.1 fails)🙁
Donald
|
{
"cluster_fingerprint": "c1bc0586-6e8c-4f1c-a92c-4286d500802e",
"version": "17.2.0",
"commit": "43e2e60a7559d3f46c9d53f1ca875fd499a1e35e",
"timestamp": "2024-03-01T20:17:04.297059+0000",
"tag": "",
"health": {
"status": "HEALTH_WARN",
"checks": {
"MGR_MODULE_DEPENDENCY": {
"severity": "HEALTH_WARN",
"summary": {
"message": "Module 'telemetry' has failed dependency: ",
"count": 1
},
"detail": [
{
"message": "Module 'telemetry' has failed dependency: "
}
],
"muted": false
},
"MON_MSGR2_NOT_ENABLED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "1 monitors have not enabled msgr2",
"count": 1
},
"detail": [
{
"message": "mon.ceph-mon1 is not bound to a msgr2 port, only v1:10.0.1.59:6789/0"
}
],
"muted": false
},
"OSD_FILESTORE": {
"severity": "HEALTH_WARN",
"summary": {
"message": "5 osd(s) are running Filestore [Deprecated]",
"count": 5
},
"detail": [
{
"message": "5 osd(s) are running Filestore, which has been deprecated and not been optimized for QoS (Filestore OSDs will use 'osd_op_queue = wpq' strictly)"
}
],
"muted": false
},
"POOL_APP_NOT_ENABLED": {
"severity": "HEALTH_WARN",
"summary": {
"message": "2 pool(s) do not have an application enabled",
"count": 2
},
"detail": [
{
"message": "application not enabled on pool 'faasm'"
},
{
"message": "application not enabled on pool 'ndp'"
},
{
"message": "use 'ceph osd pool application enable <pool-name> <app-name>', where <app-name> is 'cephfs', 'rbd', 'rgw', or freeform for custom applications."
}
],
"muted": false
}
},
"mutes": []
},
"monmap_first_committed": 1,
"monmap_last_committed": 1,
"monmap": {
"epoch": 1,
"fsid": "94ccae01-cc84-4fc0-96fd-032579767ef6",
"modified": "2024-02-28T17:12:55.152514Z",
"created": "2024-02-28T17:12:55.152514Z",
"min_mon_release": 17,
"min_mon_release_name": "quincy",
"election_strategy": 1,
"disallowed_leaders: ": "",
"stretch_mode": false,
"tiebreaker_mon": "",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus",
"octopus",
"pacific",
"elector-pinging",
"quincy"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "ceph-mon1",
"public_addrs": {
"addrvec": [
{
"type": "v1",
"addr": "10.0.1.59:6789",
"nonce": 0
}
]
},
"addr": "10.0.1.59:6789/0",
"public_addr": "10.0.1.59:6789/0",
"priority": 0,
"weight": 0,
"crush_location": "{}"
}
]
},
"quorum": [
0
],
"osdmap": {
"epoch": 51,
"fsid": "94ccae01-cc84-4fc0-96fd-032579767ef6",
"created": "2024-02-28T17:12:55.242069+0000",
"modified": "2024-03-01T18:23:28.898736+0000",
"last_up_change": "2024-02-28T20:06:17.500115+0000",
"last_in_change": "2024-02-28T20:06:17.500115+0000",
"flags": "sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit",
"flags_num": 5799936,
"flags_set": [
"pglog_hardlimit",
"purged_snapdirs",
"recovery_deletes",
"sortbitwise"
],
"crush_version": 11,
"full_ratio": 0.99000000953674316,
"backfillfull_ratio": 0.99000000953674316,
"nearfull_ratio": 0.99000000953674316,
"cluster_snapshot": "",
"pool_max": 3,
"max_osd": 5,
"require_min_compat_client": "luminous",
"min_compat_client": "jewel",
"require_osd_release": "quincy",
"pools": [
{
"pool": 1,
"pool_name": "faasm",
"create_time": "2024-02-28T17:12:55.814561+0000",
"flags": 1,
"flags_names": "hashpspool",
"type": 1,
"size": 5,
"min_size": 1,
"crush_rule": 0,
"peering_crush_bucket_count": 0,
"peering_crush_bucket_target": 0,
"peering_crush_bucket_barrier": 0,
"peering_crush_bucket_mandatory_member": 2147483647,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 32,
"pg_placement_num": 32,
"pg_placement_num_target": 32,
"pg_num_target": 32,
"pg_num_pending": 32,
"last_pg_merge_meta": {
"source_pgid": "0.0",
"ready_epoch": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"source_version": "0'0",
"target_version": "0'0"
},
"last_change": "24",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "0",
"last_force_op_resend_preluminous": "22",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 0,
"snap_epoch": 0,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {},
"application_metadata": {}
},
{
"pool": 2,
"pool_name": ".mgr",
"create_time": "2024-02-28T17:13:17.264731+0000",
"flags": 1,
"flags_names": "hashpspool",
"type": 1,
"size": 5,
"min_size": 1,
"crush_rule": 0,
"peering_crush_bucket_count": 0,
"peering_crush_bucket_target": 0,
"peering_crush_bucket_barrier": 0,
"peering_crush_bucket_mandatory_member": 2147483647,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 1,
"pg_placement_num": 1,
"pg_placement_num_target": 1,
"pg_num_target": 1,
"pg_num_pending": 1,
"last_pg_merge_meta": {
"source_pgid": "0.0",
"ready_epoch": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"source_version": "0'0",
"target_version": "0'0"
},
"last_change": "20",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "0",
"last_force_op_resend_preluminous": "0",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 0,
"snap_epoch": 0,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {
"pg_num_max": 32,
"pg_num_min": 1
},
"application_metadata": {
"mgr": {}
}
},
{
"pool": 3,
"pool_name": "ndp",
"create_time": "2024-02-28T17:17:05.136023+0000",
"flags": 1,
"flags_names": "hashpspool",
"type": 1,
"size": 5,
"min_size": 1,
"crush_rule": 0,
"peering_crush_bucket_count": 0,
"peering_crush_bucket_target": 0,
"peering_crush_bucket_barrier": 0,
"peering_crush_bucket_mandatory_member": 2147483647,
"object_hash": 2,
"pg_autoscale_mode": "on",
"pg_num": 32,
"pg_placement_num": 32,
"pg_placement_num_target": 32,
"pg_num_target": 32,
"pg_num_pending": 32,
"last_pg_merge_meta": {
"source_pgid": "0.0",
"ready_epoch": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"source_version": "0'0",
"target_version": "0'0"
},
"last_change": "32",
"last_force_op_resend": "0",
"last_force_op_resend_prenautilus": "0",
"last_force_op_resend_preluminous": "30",
"auid": 0,
"snap_mode": "selfmanaged",
"snap_seq": 0,
"snap_epoch": 0,
"pool_snaps": [],
"removed_snaps": "[]",
"quota_max_bytes": 0,
"quota_max_objects": 0,
"tiers": [],
"tier_of": -1,
"read_tier": -1,
"write_tier": -1,
"cache_mode": "none",
"target_max_bytes": 0,
"target_max_objects": 0,
"cache_target_dirty_ratio_micro": 400000,
"cache_target_dirty_high_ratio_micro": 600000,
"cache_target_full_ratio_micro": 800000,
"cache_min_flush_age": 0,
"cache_min_evict_age": 0,
"erasure_code_profile": "",
"hit_set_params": {
"type": "none"
},
"hit_set_period": 0,
"hit_set_count": 0,
"use_gmt_hitset": true,
"min_read_recency_for_promote": 0,
"min_write_recency_for_promote": 0,
"hit_set_grade_decay_rate": 0,
"hit_set_search_last_n": 0,
"grade_table": [],
"stripe_width": 0,
"expected_num_objects": 0,
"fast_read": false,
"options": {},
"application_metadata": {}
}
],
"osds": [
{
"osd": 0,
"uuid": "72c32e58-fa2e-5973-9fab-2c0e4224e839",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 0,
"last_clean_end": 0,
"up_from": 39,
"up_thru": 44,
"down_at": 36,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v1",
"addr": "10.0.1.142:6801",
"nonce": 7
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.142:6802",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.142:6803",
"nonce": 7
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.142:6806",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.142:6807",
"nonce": 7
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.142:6804",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.142:6805",
"nonce": 7
}
]
},
"public_addr": "10.0.1.142:6801/7",
"cluster_addr": "10.0.1.142:6803/7",
"heartbeat_back_addr": "10.0.1.142:6807/7",
"heartbeat_front_addr": "10.0.1.142:6805/7",
"state": [
"exists",
"up"
]
},
{
"osd": 1,
"uuid": "775b6f1a-73bd-5041-b474-6eea0e460f15",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 0,
"last_clean_end": 0,
"up_from": 44,
"up_thru": 44,
"down_at": 34,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v1",
"addr": "10.0.1.227:6801",
"nonce": 7
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.227:6802",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.227:6803",
"nonce": 7
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.227:6806",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.227:6807",
"nonce": 7
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.227:6804",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.227:6805",
"nonce": 7
}
]
},
"public_addr": "10.0.1.227:6801/7",
"cluster_addr": "10.0.1.227:6803/7",
"heartbeat_back_addr": "10.0.1.227:6807/7",
"heartbeat_front_addr": "10.0.1.227:6805/7",
"state": [
"exists",
"up"
]
},
{
"osd": 2,
"uuid": "fe3a2694-229f-5184-b7d3-511400ed81a1",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 0,
"last_clean_end": 0,
"up_from": 12,
"up_thru": 44,
"down_at": 0,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v1",
"addr": "10.0.1.43:6801",
"nonce": 8
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.43:6802",
"nonce": 8
},
{
"type": "v1",
"addr": "10.0.1.43:6803",
"nonce": 8
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.43:6806",
"nonce": 8
},
{
"type": "v1",
"addr": "10.0.1.43:6807",
"nonce": 8
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.43:6804",
"nonce": 8
},
{
"type": "v1",
"addr": "10.0.1.43:6805",
"nonce": 8
}
]
},
"public_addr": "10.0.1.43:6801/8",
"cluster_addr": "10.0.1.43:6803/8",
"heartbeat_back_addr": "10.0.1.43:6807/8",
"heartbeat_front_addr": "10.0.1.43:6805/8",
"state": [
"exists",
"up"
]
},
{
"osd": 3,
"uuid": "4b05d63a-1716-53d2-a3c6-541f462e8af7",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 0,
"last_clean_end": 0,
"up_from": 13,
"up_thru": 44,
"down_at": 0,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v1",
"addr": "10.0.1.49:6801",
"nonce": 7
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.49:6802",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.49:6803",
"nonce": 7
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.49:6806",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.49:6807",
"nonce": 7
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.49:6804",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.49:6805",
"nonce": 7
}
]
},
"public_addr": "10.0.1.49:6801/7",
"cluster_addr": "10.0.1.49:6803/7",
"heartbeat_back_addr": "10.0.1.49:6807/7",
"heartbeat_front_addr": "10.0.1.49:6805/7",
"state": [
"exists",
"up"
]
},
{
"osd": 4,
"uuid": "feb7d10a-8a17-51e7-b3f3-d2f83917526e",
"up": 1,
"in": 1,
"weight": 1,
"primary_affinity": 1,
"last_clean_begin": 0,
"last_clean_end": 0,
"up_from": 17,
"up_thru": 44,
"down_at": 0,
"lost_at": 0,
"public_addrs": {
"addrvec": [
{
"type": "v1",
"addr": "10.0.1.47:6801",
"nonce": 7
}
]
},
"cluster_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.47:6802",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.47:6803",
"nonce": 7
}
]
},
"heartbeat_back_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.47:6806",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.47:6807",
"nonce": 7
}
]
},
"heartbeat_front_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "10.0.1.47:6804",
"nonce": 7
},
{
"type": "v1",
"addr": "10.0.1.47:6805",
"nonce": 7
}
]
},
"public_addr": "10.0.1.47:6801/7",
"cluster_addr": "10.0.1.47:6803/7",
"heartbeat_back_addr": "10.0.1.47:6807/7",
"heartbeat_front_addr": "10.0.1.47:6805/7",
"state": [
"exists",
"up"
]
}
],
"osd_xinfo": [
{
"osd": 0,
"down_stamp": "2024-02-28T20:00:35.073448+0000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138303579357183,
"old_weight": 0,
"last_purged_snaps_scrub": "2024-03-01T03:38:51.182599+0000",
"dead_epoch": 0
},
{
"osd": 1,
"down_stamp": "2024-02-28T19:56:13.307900+0000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138303579357183,
"old_weight": 0,
"last_purged_snaps_scrub": "2024-02-29T18:35:24.406590+0000",
"dead_epoch": 35
},
{
"osd": 2,
"down_stamp": "0.000000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138303579357183,
"old_weight": 0,
"last_purged_snaps_scrub": "2024-02-29T17:38:28.300282+0000",
"dead_epoch": 0
},
{
"osd": 3,
"down_stamp": "0.000000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138303579357183,
"old_weight": 0,
"last_purged_snaps_scrub": "2024-03-01T18:23:28.326020+0000",
"dead_epoch": 0
},
{
"osd": 4,
"down_stamp": "0.000000",
"laggy_probability": 0,
"laggy_interval": 0,
"features": 4540138303579357183,
"old_weight": 0,
"last_purged_snaps_scrub": "2024-02-29T17:26:37.623964+0000",
"dead_epoch": 0
}
],
"pg_upmap": [],
"pg_upmap_items": [],
"pg_temp": [],
"primary_temp": [],
"blocklist": {},
"erasure_code_profiles": {
"default": {
"k": "2",
"m": "2",
"plugin": "jerasure",
"technique": "reed_sol_van"
}
},
"removed_snaps_queue": [],
"new_removed_snaps": [],
"new_purged_snaps": [],
"crush_node_flags": {},
"device_class_flags": {},
"stretch_mode": {
"stretch_mode_enabled": false,
"stretch_bucket_count": 0,
"degraded_stretch_mode": 0,
"recovering_stretch_mode": 0,
"stretch_mode_bucket": 0
}
},
"osd_metadata": [
{
"id": 0,
"arch": "x86_64",
"back_addr": "[v2:10.0.1.142:6802/7,v1:10.0.1.142:6803/7]",
"back_iface": "",
"backend_filestore_dev_node": "loop30",
"backend_filestore_journal_dev_node": "loop30",
"backend_filestore_journal_partition_path": "/dev/loop30",
"backend_filestore_partition_path": "/dev/loop30",
"ceph_release": "quincy",
"ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)",
"ceph_version_short": "17.2.0",
"cpu": "AMD EPYC 7302P 16-Core Processor",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "",
"devices": "loop30",
"distro": "ubuntu",
"distro_description": "Ubuntu 22.04.1 LTS",
"distro_version": "22.04",
"filestore_backend": "generic",
"filestore_f_type": "0xef53",
"front_addr": "[v2:10.0.1.142:6800/7,v1:10.0.1.142:6801/7]",
"front_iface": "",
"hb_back_addr": "[v2:10.0.1.142:6806/7,v1:10.0.1.142:6807/7]",
"hb_front_addr": "[v2:10.0.1.142:6804/7,v1:10.0.1.142:6805/7]",
"hostname": "ceph-osd1",
"journal_rotational": "0",
"kernel_description": "#181-Ubuntu SMP Fri Sep 1 13:41:22 UTC 2023",
"kernel_version": "5.4.0-164-generic",
"mem_swap_kb": "0",
"mem_total_kb": "131563656",
"network_numa_unknown_ifaces": "back_iface,front_iface",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-0",
"osd_journal": "/var/lib/ceph/osd/ceph-0/journal",
"osd_objectstore": "filestore",
"osdspec_affinity": "",
"rotational": "0"
},
{
"id": 1,
"arch": "x86_64",
"back_addr": "[v2:10.0.1.227:6802/7,v1:10.0.1.227:6803/7]",
"back_iface": "",
"backend_filestore_dev_node": "loop30",
"backend_filestore_journal_dev_node": "loop30",
"backend_filestore_journal_partition_path": "/dev/loop30",
"backend_filestore_partition_path": "/dev/loop30",
"ceph_release": "quincy",
"ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)",
"ceph_version_short": "17.2.0",
"cpu": "AMD EPYC 7302P 16-Core Processor",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "",
"devices": "loop30",
"distro": "ubuntu",
"distro_description": "Ubuntu 22.04.1 LTS",
"distro_version": "22.04",
"filestore_backend": "generic",
"filestore_f_type": "0xef53",
"front_addr": "[v2:10.0.1.227:6800/7,v1:10.0.1.227:6801/7]",
"front_iface": "",
"hb_back_addr": "[v2:10.0.1.227:6806/7,v1:10.0.1.227:6807/7]",
"hb_front_addr": "[v2:10.0.1.227:6804/7,v1:10.0.1.227:6805/7]",
"hostname": "ceph-osd2",
"journal_rotational": "0",
"kernel_description": "#181-Ubuntu SMP Fri Sep 1 13:41:22 UTC 2023",
"kernel_version": "5.4.0-164-generic",
"mem_swap_kb": "0",
"mem_total_kb": "131563656",
"network_numa_unknown_ifaces": "back_iface,front_iface",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-1",
"osd_journal": "/var/lib/ceph/osd/ceph-1/journal",
"osd_objectstore": "filestore",
"osdspec_affinity": "",
"rotational": "0"
},
{
"id": 2,
"arch": "x86_64",
"back_addr": "[v2:10.0.1.43:6802/8,v1:10.0.1.43:6803/8]",
"back_iface": "",
"backend_filestore_dev_node": "loop30",
"backend_filestore_journal_dev_node": "loop30",
"backend_filestore_journal_partition_path": "/dev/loop30",
"backend_filestore_partition_path": "/dev/loop30",
"ceph_release": "quincy",
"ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)",
"ceph_version_short": "17.2.0",
"cpu": "AMD EPYC 7302P 16-Core Processor",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "",
"devices": "loop30",
"distro": "ubuntu",
"distro_description": "Ubuntu 22.04.1 LTS",
"distro_version": "22.04",
"filestore_backend": "generic",
"filestore_f_type": "0xef53",
"front_addr": "[v2:10.0.1.43:6800/8,v1:10.0.1.43:6801/8]",
"front_iface": "",
"hb_back_addr": "[v2:10.0.1.43:6806/8,v1:10.0.1.43:6807/8]",
"hb_front_addr": "[v2:10.0.1.43:6804/8,v1:10.0.1.43:6805/8]",
"hostname": "ceph-osd3",
"journal_rotational": "0",
"kernel_description": "#181-Ubuntu SMP Fri Sep 1 13:41:22 UTC 2023",
"kernel_version": "5.4.0-164-generic",
"mem_swap_kb": "0",
"mem_total_kb": "131563656",
"network_numa_unknown_ifaces": "back_iface,front_iface",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-2",
"osd_journal": "/var/lib/ceph/osd/ceph-2/journal",
"osd_objectstore": "filestore",
"osdspec_affinity": "",
"rotational": "0"
},
{
"id": 3,
"arch": "x86_64",
"back_addr": "[v2:10.0.1.49:6802/7,v1:10.0.1.49:6803/7]",
"back_iface": "",
"backend_filestore_dev_node": "loop30",
"backend_filestore_journal_dev_node": "loop30",
"backend_filestore_journal_partition_path": "/dev/loop30",
"backend_filestore_partition_path": "/dev/loop30",
"ceph_release": "quincy",
"ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)",
"ceph_version_short": "17.2.0",
"cpu": "AMD EPYC 7302P 16-Core Processor",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "",
"devices": "loop30",
"distro": "ubuntu",
"distro_description": "Ubuntu 22.04.1 LTS",
"distro_version": "22.04",
"filestore_backend": "generic",
"filestore_f_type": "0xef53",
"front_addr": "[v2:10.0.1.49:6800/7,v1:10.0.1.49:6801/7]",
"front_iface": "",
"hb_back_addr": "[v2:10.0.1.49:6806/7,v1:10.0.1.49:6807/7]",
"hb_front_addr": "[v2:10.0.1.49:6804/7,v1:10.0.1.49:6805/7]",
"hostname": "ceph-osd4",
"journal_rotational": "0",
"kernel_description": "#181-Ubuntu SMP Fri Sep 1 13:41:22 UTC 2023",
"kernel_version": "5.4.0-164-generic",
"mem_swap_kb": "0",
"mem_total_kb": "131563656",
"network_numa_unknown_ifaces": "back_iface,front_iface",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-3",
"osd_journal": "/var/lib/ceph/osd/ceph-3/journal",
"osd_objectstore": "filestore",
"osdspec_affinity": "",
"rotational": "0"
},
{
"id": 4,
"arch": "x86_64",
"back_addr": "[v2:10.0.1.47:6802/7,v1:10.0.1.47:6803/7]",
"back_iface": "",
"backend_filestore_dev_node": "loop30",
"backend_filestore_journal_dev_node": "loop30",
"backend_filestore_journal_partition_path": "/dev/loop30",
"backend_filestore_partition_path": "/dev/loop30",
"ceph_release": "quincy",
"ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)",
"ceph_version_short": "17.2.0",
"cpu": "AMD EPYC 7302P 16-Core Processor",
"default_device_class": "ssd",
"device_ids": "",
"device_paths": "",
"devices": "loop30",
"distro": "ubuntu",
"distro_description": "Ubuntu 22.04.1 LTS",
"distro_version": "22.04",
"filestore_backend": "generic",
"filestore_f_type": "0xef53",
"front_addr": "[v2:10.0.1.47:6800/7,v1:10.0.1.47:6801/7]",
"front_iface": "",
"hb_back_addr": "[v2:10.0.1.47:6806/7,v1:10.0.1.47:6807/7]",
"hb_front_addr": "[v2:10.0.1.47:6804/7,v1:10.0.1.47:6805/7]",
"hostname": "ceph-osd5",
"journal_rotational": "0",
"kernel_description": "#181-Ubuntu SMP Fri Sep 1 13:41:22 UTC 2023",
"kernel_version": "5.4.0-164-generic",
"mem_swap_kb": "0",
"mem_total_kb": "131563656",
"network_numa_unknown_ifaces": "back_iface,front_iface",
"os": "Linux",
"osd_data": "/var/lib/ceph/osd/ceph-4",
"osd_journal": "/var/lib/ceph/osd/ceph-4/journal",
"osd_objectstore": "filestore",
"osdspec_affinity": "",
"rotational": "0"
}
],
"osdmap_clean_epochs": {
"min_last_epoch_clean": 51,
"last_epoch_clean": {
"per_pool": [
{
"poolid": 1,
"floor": 51
},
{
"poolid": 2,
"floor": 51
},
{
"poolid": 3,
"floor": 51
}
]
},
"osd_epochs": [
{
"id": 0,
"epoch": 51
},
{
"id": 1,
"epoch": 51
},
{
"id": 2,
"epoch": 51
},
{
"id": 3,
"epoch": 51
},
{
"id": 4,
"epoch": 51
}
]
},
"osdmap_first_committed": 1,
"osdmap_last_committed": 51,
"crushmap": {
"devices": [
{
"id": 0,
"name": "osd.0",
"class": "ssd"
},
{
"id": 1,
"name": "osd.1",
"class": "ssd"
},
{
"id": 2,
"name": "osd.2",
"class": "ssd"
},
{
"id": 3,
"name": "osd.3",
"class": "ssd"
},
{
"id": 4,
"name": "osd.4",
"class": "ssd"
}
],
"types": [
{
"type_id": 0,
"name": "osd"
},
{
"type_id": 1,
"name": "host"
},
{
"type_id": 2,
"name": "chassis"
},
{
"type_id": 3,
"name": "rack"
},
{
"type_id": 4,
"name": "row"
},
{
"type_id": 5,
"name": "pdu"
},
{
"type_id": 6,
"name": "pod"
},
{
"type_id": 7,
"name": "room"
},
{
"type_id": 8,
"name": "datacenter"
},
{
"type_id": 9,
"name": "zone"
},
{
"type_id": 10,
"name": "region"
},
{
"type_id": 11,
"name": "root"
}
],
"buckets": [
{
"id": -1,
"name": "default",
"type_id": 11,
"type_name": "root",
"weight": 25030,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -3,
"weight": 5006,
"pos": 0
},
{
"id": -5,
"weight": 5006,
"pos": 1
},
{
"id": -7,
"weight": 5006,
"pos": 2
},
{
"id": -9,
"weight": 5006,
"pos": 3
},
{
"id": -11,
"weight": 5006,
"pos": 4
}
]
},
{
"id": -2,
"name": "default~ssd",
"type_id": 11,
"type_name": "root",
"weight": 25030,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": -4,
"weight": 5006,
"pos": 0
},
{
"id": -6,
"weight": 5006,
"pos": 1
},
{
"id": -8,
"weight": 5006,
"pos": 2
},
{
"id": -10,
"weight": 5006,
"pos": 3
},
{
"id": -12,
"weight": 5006,
"pos": 4
}
]
},
{
"id": -3,
"name": "ceph-osd1",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -4,
"name": "ceph-osd1~ssd",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 0,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -5,
"name": "ceph-osd2",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 1,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -6,
"name": "ceph-osd2~ssd",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 1,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -7,
"name": "ceph-osd3",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 2,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -8,
"name": "ceph-osd3~ssd",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 2,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -9,
"name": "ceph-osd4",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 3,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -10,
"name": "ceph-osd4~ssd",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 3,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -11,
"name": "ceph-osd5",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 4,
"weight": 5006,
"pos": 0
}
]
},
{
"id": -12,
"name": "ceph-osd5~ssd",
"type_id": 1,
"type_name": "host",
"weight": 5006,
"alg": "straw2",
"hash": "rjenkins1",
"items": [
{
"id": 4,
"weight": 5006,
"pos": 0
}
]
}
],
"rules": [
{
"rule_id": 0,
"rule_name": "replicated_rule",
"type": 1,
"steps": [
{
"op": "take",
"item": -1,
"item_name": "default"
},
{
"op": "choose_firstn",
"num": 0,
"type": "osd"
},
{
"op": "emit"
}
]
}
],
"tunables": {
"choose_local_tries": 0,
"choose_local_fallback_tries": 0,
"choose_total_tries": 50,
"chooseleaf_descend_once": 1,
"chooseleaf_vary_r": 1,
"chooseleaf_stable": 1,
"straw_calc_version": 1,
"allowed_bucket_algs": 54,
"profile": "jewel",
"optimal_tunables": 1,
"legacy_tunables": 0,
"minimum_required_version": "jewel",
"require_feature_tunables": 1,
"require_feature_tunables2": 1,
"has_v2_rules": 0,
"require_feature_tunables3": 1,
"has_v3_rules": 0,
"has_v4_buckets": 1,
"require_feature_tunables5": 1,
"has_v5_rules": 0
},
"choose_args": {}
},
"fsmap": {
"epoch": 1,
"default_fscid": -1,
"compat": {
"compat": {},
"ro_compat": {},
"incompat": {
"feature_1": "base v0.20",
"feature_2": "client writeable ranges",
"feature_3": "default file layouts on dirs",
"feature_4": "dir inode in separate object",
"feature_5": "mds uses versioned encoding",
"feature_6": "dirfrag is stored in omap",
"feature_8": "no anchor table",
"feature_9": "file layout v2",
"feature_10": "snaprealm v2"
}
},
"feature_flags": {
"enable_multiple": true,
"ever_enabled_multiple": true
},
"standbys": [],
"filesystems": []
},
"mdsmap_first_committed": 1,
"mdsmap_last_committed": 1,
"auth": {
"first_committed": 1,
"last_committed": 58,
"num_secrets": 13
},
"num_pg": 65,
"num_pg_active": 65,
"num_pg_unknown": 0,
"num_osd": 5,
"pool_sum": {
"stat_sum": {
"num_bytes": 164645129,
"num_objects": 33,
"num_object_clones": 0,
"num_object_copies": 165,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 33,
"num_whiteouts": 0,
"num_read": 979362,
"num_read_kb": 14691220568,
"num_write": 284,
"num_write_kb": 172951,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1,
"num_bytes_recovered": 4,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 0,
"data_stored": 0,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 669,
"ondisk_log_size": 669,
"up": 325,
"acting": 325,
"num_store_stats": 0
},
"osd_sum": {
"up_from": 0,
"seq": 0,
"num_pgs": 325,
"num_osds": 5,
"num_per_pool_osds": 0,
"num_per_pool_omap_osds": 0,
"kb": 409939960,
"kb_used": 27491560,
"kb_used_data": 382448400,
"kb_used_omap": 9410,
"kb_used_meta": 0,
"kb_avail": 382448400,
"statfs": {
"total": 419778519040,
"available": 391627161600,
"internally_reserved": 0,
"allocated": 391627161600,
"data_stored": 391627161600,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 9635915,
"internal_metadata": 0
},
"hb_peers": [],
"snap_trim_queue_len": 0,
"num_snap_trimming": 0,
"num_shards_repaired": 0,
"op_queue_age_hist": {
"histogram": [],
"upper_bound": 1
},
"perf_stat": {
"commit_latency_ms": 0,
"apply_latency_ms": 0,
"commit_latency_ns": 0,
"apply_latency_ns": 0
},
"alerts": [],
"network_ping_times": []
},
"osd_sum_by_class": {
"ssd": {
"up_from": 0,
"seq": 0,
"num_pgs": 325,
"num_osds": 5,
"num_per_pool_osds": 0,
"num_per_pool_omap_osds": 0,
"kb": 409939960,
"kb_used": 27491560,
"kb_used_data": 382448400,
"kb_used_omap": 9410,
"kb_used_meta": 0,
"kb_avail": 382448400,
"statfs": {
"total": 419778519040,
"available": 391627161600,
"internally_reserved": 0,
"allocated": 391627161600,
"data_stored": 391627161600,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 9635915,
"internal_metadata": 0
},
"hb_peers": [],
"snap_trim_queue_len": 0,
"num_snap_trimming": 0,
"num_shards_repaired": 0,
"op_queue_age_hist": {
"histogram": [],
"upper_bound": 1
},
"perf_stat": {
"commit_latency_ms": 0,
"apply_latency_ms": 0,
"commit_latency_ns": 0,
"apply_latency_ns": 0
},
"alerts": [],
"network_ping_times": []
}
},
"pool_stats": [
{
"poolid": 3,
"num_pg": 32,
"stat_sum": {
"num_bytes": 163264628,
"num_objects": 10,
"num_object_clones": 0,
"num_object_copies": 50,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 10,
"num_whiteouts": 0,
"num_read": 979046,
"num_read_kb": 14691217836,
"num_write": 53,
"num_write_kb": 170073,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 0,
"data_stored": 0,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 88,
"ondisk_log_size": 88,
"up": 160,
"acting": 160,
"num_store_stats": 0
},
{
"poolid": 2,
"num_pg": 1,
"stat_sum": {
"num_bytes": 459280,
"num_objects": 2,
"num_object_clones": 0,
"num_object_copies": 10,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 2,
"num_whiteouts": 0,
"num_read": 116,
"num_read_kb": 100,
"num_write": 141,
"num_write_kb": 1896,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 0,
"data_stored": 0,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 100,
"ondisk_log_size": 100,
"up": 5,
"acting": 5,
"num_store_stats": 0
},
{
"poolid": 1,
"num_pg": 32,
"stat_sum": {
"num_bytes": 921221,
"num_objects": 21,
"num_object_clones": 0,
"num_object_copies": 105,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 21,
"num_whiteouts": 0,
"num_read": 200,
"num_read_kb": 2632,
"num_write": 90,
"num_write_kb": 982,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1,
"num_bytes_recovered": 4,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"store_stats": {
"total": 0,
"available": 0,
"internally_reserved": 0,
"allocated": 0,
"data_stored": 0,
"data_compressed": 0,
"data_compressed_allocated": 0,
"data_compressed_original": 0,
"omap_allocated": 0,
"internal_metadata": 0
},
"log_size": 481,
"ondisk_log_size": 481,
"up": 160,
"acting": 160,
"num_store_stats": 0
}
],
"osd_stats": [
{
"osd": 0,
"seq": 34734
},
{
"osd": 1,
"seq": 34670
},
{
"osd": 2,
"seq": 36761
},
{
"osd": 3,
"seq": 36767
},
{
"osd": 4,
"seq": 36758
}
],
"num_pg_by_state": [
{
"state": "active+clean",
"num": 65
}
],
"num_pg_by_osd": [
{
"osd": 4,
"num_primary_pg": 8,
"num_acting_pg": 65,
"num_up_not_acting_pg": 0
},
{
"osd": 1,
"num_primary_pg": 15,
"num_acting_pg": 65,
"num_up_not_acting_pg": 0
},
{
"osd": 2,
"num_primary_pg": 14,
"num_acting_pg": 65,
"num_up_not_acting_pg": 0
},
{
"osd": 3,
"num_primary_pg": 16,
"num_acting_pg": 65,
"num_up_not_acting_pg": 0
},
{
"osd": 0,
"num_primary_pg": 12,
"num_acting_pg": 65,
"num_up_not_acting_pg": 0
}
],
"purged_snaps": [
{
"pool": 1,
"purged_snaps": {}
},
{
"pool": 2,
"purged_snaps": {}
},
{
"pool": 3,
"purged_snaps": {}
}
],
"servicemap": {
"epoch": 1163,
"modified": "2024-03-01T20:14:19.304581+0000",
"services": {}
},
"paxos": {
"first_committed": 223391,
"last_committed": 223961,
"last_pn": 0,
"accepted_pn": 0
}
}_______________________________________________ Dev mailing list -- dev@xxxxxxx To unsubscribe send an email to dev-leave@xxxxxxx
