Hi community
We have a new Ceph cluster deployment with 100 nodes. When we are draining an OSD host from the cluster, we see a small amount of PGs that cannot make any progress to the end. From the logs and metrics, it seems like the recovery progress is stuck (0 recovery ops for several days). Would like to get some ideas on this. Re-peering and OSD restart do resolve to mitigate the issue but we want to get to the root cause of it as draining and recovery happen frequently.
I have put some debugging information below. Any help is appreciated, thanks!
ceph -s
pgs: 4210926/7380034104 objects misplaced (0.057%)
41198 active+clean
71 active+remapped+backfilling
12 active+recovering
One of the stuck PG:
6.38f1 active+remapped+backfilling [313,643,727] 313 [313,643,717] 313
PG query result:
ceph pg 6.38f1 query
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "active+remapped+backfilling",
"epoch": 246856,
"up": [
313,
643,
727
],
"acting": [
313,
643,
717
],
"backfill_targets": [
"727"
],
"acting_recovery_backfill": [
"313",
"643",
"717",
"727"
],
"info": {
"pgid": "6.38f1",
"last_update": "212333'38916",
"last_complete": "212333'38916",
"log_tail": "80608'37589",
"last_user_version": 38833,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 3726,
"epoch_pool_created": 3279,
"last_epoch_started": 243987,
"last_interval_started": 243986,
"last_epoch_clean": 220174,
"last_interval_clean": 220173,
"last_epoch_split": 3726,
"last_epoch_marked_full": 0,
"same_up_since": 238347,
"same_interval_since": 243986,
"same_primary_since": 3728,
"last_scrub": "212333'38916",
"last_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"last_deep_scrub": "212333'38916",
"last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000",
"last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "212333'38916",
"reported_seq": 413425,
"reported_epoch": 246856,
"state": "active+remapped+backfilling",
"last_fresh": "2024-02-05T21:14:40.838785+0000",
"last_change": "2024-02-03T22:33:43.052272+0000",
"last_active": "2024-02-05T21:14:40.838785+0000",
"last_peered": "2024-02-05T21:14:40.838785+0000",
"last_clean": "2024-02-03T04:26:35.168232+0000",
"last_became_active": "2024-02-03T22:31:16.037823+0000",
"last_became_peered": "2024-02-03T22:31:16.037823+0000",
"last_unstale": "2024-02-05T21:14:40.838785+0000",
"last_undegraded": "2024-02-05T21:14:40.838785+0000",
"last_fullsized": "2024-02-05T21:14:40.838785+0000",
"mapping_epoch": 243986,
"log_start": "80608'37589",
"ondisk_log_start": "80608'37589",
"created": 3726,
"last_epoch_clean": 220174,
"parent": "0.0",
"parent_split_bits": 14,
"last_scrub": "212333'38916",
"last_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"last_deep_scrub": "212333'38916",
"last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000",
"last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"objects_scrubbed": 17743,
"log_size": 1327,
"log_dups_size": 3000,
"ondisk_log_size": 1327,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 29,
"scrub_schedule": "queued for deep scrub",
"scrub_duration": 28.699844475999999,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 67167022618,
"num_objects": 17743,
"num_object_clones": 0,
"num_object_copies": 53229,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 12401,
"num_objects_unfound": 0,
"num_objects_dirty": 17743,
"num_whiteouts": 0,
"num_read": 16640,
"num_read_kb": 30923655,
"num_write": 55094,
"num_write_kb": 71824558,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 40746,
"num_bytes_recovered": 154245233300,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
313,
643,
727
],
"acting": [
313,
643,
717
],
"avail_no_missing": [
"313",
"643",
"717"
],
"object_location_counts": [
{
"shards": "313,643,717",
"objects": 17743
}
],
"blocked_by": [],
"up_primary": 313,
"acting_primary": 313,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 243987,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "643",
"pgid": "6.38f1",
"last_update": "212333'38916",
"last_complete": "212333'38916",
"log_tail": "80608'37589",
"last_user_version": 38833,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 3726,
"epoch_pool_created": 3279,
"last_epoch_started": 243987,
"last_interval_started": 243986,
"last_epoch_clean": 220174,
"last_interval_clean": 220173,
"last_epoch_split": 3726,
"last_epoch_marked_full": 0,
"same_up_since": 238347,
"same_interval_since": 243986,
"same_primary_since": 3728,
"last_scrub": "212333'38916",
"last_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"last_deep_scrub": "212333'38916",
"last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000",
"last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "212331'38915",
"reported_seq": 362849,
"reported_epoch": 212333,
"state": "active+clean+scrubbing+deep",
"last_fresh": "2024-01-28T07:43:40.515729+0000",
"last_change": "2024-01-28T06:20:14.969313+0000",
"last_active": "2024-01-28T07:43:40.515729+0000",
"last_peered": "2024-01-28T07:43:40.515729+0000",
"last_clean": "2024-01-28T07:43:40.515729+0000",
"last_became_active": "2024-01-23T10:01:07.710478+0000",
"last_became_peered": "2024-01-23T10:01:07.710478+0000",
"last_unstale": "2024-01-28T07:43:40.515729+0000",
"last_undegraded": "2024-01-28T07:43:40.515729+0000",
"last_fullsized": "2024-01-28T07:43:40.515729+0000",
"mapping_epoch": 243986,
"log_start": "80608'37589",
"ondisk_log_start": "80608'37589",
"created": 3726,
"last_epoch_clean": 123203,
"parent": "0.0",
"parent_split_bits": 14,
"last_scrub": "80609'38697",
"last_scrub_stamp": "2024-01-09T20:27:40.795531+0000",
"last_deep_scrub": "80609'38697",
"last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000",
"last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000",
"objects_scrubbed": 17724,
"log_size": 1326,
"log_dups_size": 3000,
"ondisk_log_size": 1326,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 29,
"scrub_schedule": "deep scrubbing for 5005s",
"scrub_duration": 28.446739284,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 67167022618,
"num_objects": 17743,
"num_object_clones": 0,
"num_object_copies": 53229,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 17743,
"num_whiteouts": 0,
"num_read": 16640,
"num_read_kb": 30923655,
"num_write": 55094,
"num_write_kb": 71824558,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 35403,
"num_bytes_recovered": 134021819371,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
313,
643,
727
],
"acting": [
313,
643,
717
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 313,
"acting_primary": 313,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 243987,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "717",
"pgid": "6.38f1",
"last_update": "212333'38916",
"last_complete": "212333'38916",
"log_tail": "80608'37589",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 3726,
"epoch_pool_created": 3279,
"last_epoch_started": 243987,
"last_interval_started": 243986,
"last_epoch_clean": 220174,
"last_interval_clean": 220173,
"last_epoch_split": 3726,
"last_epoch_marked_full": 0,
"same_up_since": 238347,
"same_interval_since": 243986,
"same_primary_since": 3728,
"last_scrub": "212333'38916",
"last_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"last_deep_scrub": "212333'38916",
"last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000",
"last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "212331'38915",
"reported_seq": 362849,
"reported_epoch": 212333,
"state": "active+clean+scrubbing+deep",
"last_fresh": "2024-01-28T07:43:40.515729+0000",
"last_change": "2024-01-28T06:20:14.969313+0000",
"last_active": "2024-01-28T07:43:40.515729+0000",
"last_peered": "2024-01-28T07:43:40.515729+0000",
"last_clean": "2024-01-28T07:43:40.515729+0000",
"last_became_active": "2024-01-23T10:01:07.710478+0000",
"last_became_peered": "2024-01-23T10:01:07.710478+0000",
"last_unstale": "2024-01-28T07:43:40.515729+0000",
"last_undegraded": "2024-01-28T07:43:40.515729+0000",
"last_fullsized": "2024-01-28T07:43:40.515729+0000",
"mapping_epoch": 243986,
"log_start": "80608'37589",
"ondisk_log_start": "80608'37589",
"created": 3726,
"last_epoch_clean": 123203,
"parent": "0.0",
"parent_split_bits": 14,
"last_scrub": "80609'38697",
"last_scrub_stamp": "2024-01-09T20:27:40.795531+0000",
"last_deep_scrub": "80609'38697",
"last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000",
"last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000",
"objects_scrubbed": 17724,
"log_size": 1326,
"log_dups_size": 3000,
"ondisk_log_size": 1326,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 29,
"scrub_schedule": "deep scrubbing for 5005s",
"scrub_duration": 28.446739284,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 67167022618,
"num_objects": 17743,
"num_object_clones": 0,
"num_object_copies": 53229,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 17743,
"num_whiteouts": 0,
"num_read": 16640,
"num_read_kb": 30923655,
"num_write": 55094,
"num_write_kb": 71824558,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 35403,
"num_bytes_recovered": 134021819371,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
313,
643,
727
],
"acting": [
313,
643,
717
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 313,
"acting_primary": 313,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 243987,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "726",
"pgid": "6.38f1",
"last_update": "212333'38916",
"last_complete": "212333'38916",
"log_tail": "80608'37589",
"last_user_version": 0,
"last_backfill": "MIN",
"purged_snaps": [],
"history": {
"epoch_created": 3726,
"epoch_pool_created": 3279,
"last_epoch_started": 238348,
"last_interval_started": 238347,
"last_epoch_clean": 220174,
"last_interval_clean": 220173,
"last_epoch_split": 3726,
"last_epoch_marked_full": 0,
"same_up_since": 238347,
"same_interval_since": 243986,
"same_primary_since": 3728,
"last_scrub": "212333'38916",
"last_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"last_deep_scrub": "212333'38916",
"last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000",
"last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"prior_readable_until_ub": 11.834917336
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 243986,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"log_dups_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 4194304,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 17743,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
313,
643,
727
],
"acting": [
313,
643,
717
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 313,
"acting_primary": 313,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 238320,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "727",
"pgid": "6.38f1",
"last_update": "212333'38916",
"last_complete": "212333'38916",
"log_tail": "80608'37589",
"last_user_version": 0,
"last_backfill": "6:8f1d3293:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.V-JlvDFjbQNgn8GtBLN2LePecv5fLKf_3:head",
"purged_snaps": [],
"history": {
"epoch_created": 3726,
"epoch_pool_created": 3279,
"last_epoch_started": 243987,
"last_interval_started": 243986,
"last_epoch_clean": 220174,
"last_interval_clean": 220173,
"last_epoch_split": 3726,
"last_epoch_marked_full": 0,
"same_up_since": 238347,
"same_interval_since": 243986,
"same_primary_since": 3728,
"last_scrub": "212333'38916",
"last_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"last_deep_scrub": "212333'38916",
"last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000",
"last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 243986,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"objects_scrubbed": 0,
"log_size": 0,
"log_dups_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"last_scrub_duration": 0,
"scrub_schedule": "--",
"scrub_duration": 0,
"objects_trimmed": 0,
"snaptrim_duration": 0,
"stat_sum": {
"num_bytes": 20227608233,
"num_objects": 5342,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 12401,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 5342,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
313,
643,
727
],
"acting": [
313,
643,
717
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 313,
"acting_primary": 313,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 243987,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2024-02-03T22:31:15.023323+0000",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [
"727"
],
"waiting_on_backfill": [],
"last_backfill_started": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head",
"backfill_info": {
"begin": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head",
"end": "6:8f1d3549:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.PTxEYjlIXk2vkWpRgKUOu4ygweU-Jqy_5:head",
"objects": [
{
"object": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head",
"version": "80588'26367"
},
{
"object": "6:8f1d32b0:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_18-71435:head",
"version": "80588'26368"
},
{
"object": "6:8f1d32c2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.o_s0NGR7mrH-4JKpvYS6Ip14hbm4w5o_5:head",
"version": "80588'26369"
},
{
"object": "6:8f1d32c3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WV1Ja5V21BKfmI5e1W90a7iQXdcTnSG_6:head",
"version": "80588'26370"
},
{
"object": "6:8f1d32c4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.96tkpnCzKmtnPTLewpai2cXSr_z_QcR_1:head",
"version": "80588'26371"
},
{
"object": "6:8f1d331d:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.InACTSlwV7irTe0g9cLnadT8EbQr4Cl_5:head",
"version": "80588'26372"
},
{
"object": "6:8f1d3326:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_233-155987:head",
"version": "80588'26373"
},
{
"object": "6:8f1d3332:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.5aipEOj0vV3h4jdSf6tkAUn64yJmvoH_1:head",
"version": "80588'26374"
},
{
"object": "6:8f1d3343:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QTzZJhDV6JLC5wJ5BDxHvK4sfp3Hr6o_6:head",
"version": "80588'26375"
},
{
"object": "6:8f1d334b:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QgMCmHWV87HlAjqkfvbO9X5OUZrgIPc_2:head",
"version": "80588'26376"
},
{
"object": "6:8f1d3355:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.H4j6DWgQa0jsrl-xzgugsW22GHCOgGL_7:head",
"version": "80588'26377"
},
{
"object": "6:8f1d3379:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.25g7c0w-VnGqX-z_m53YmnoWUO0R56N_5:head",
"version": "80588'26378"
},
{
"object": "6:8f1d337c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WMmV4BoLMm4q6BztoZ1uPc4boiJ-zhW_1:head",
"version": "80588'26379"
},
{
"object": "6:8f1d3385:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_184-108817:head",
"version": "80588'26380"
},
{
"object": "6:8f1d3389:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sGNiGV9K-TGyTYRqWSOcSIXkBBOT93X_3:head",
"version": "80588'26381"
},
{
"object": "6:8f1d339c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.NkMm_dhUxRyxXhBDYqGCuOaSvwNvSuJ_3:head",
"version": "80588'26382"
},
{
"object": "6:8f1d33a3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.BV__z8jIWqcjMsTf40MvaARU1dVtCv2_4:head",
"version": "80588'26383"
},
{
"object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.XJLSyRSnrDupnOjWzHKc4wykPJjqR5g_1:head",
"version": "80588'26384"
},
{
"object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.oYSnMakOIpFmfLdym2oCx-fLBDCLDmr_4:head",
"version": "80588'26385"
},
{
"object": "6:8f1d33b4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_135-2779:head",
"version": "80588'26386"
},
{
"object": "6:8f1d33b6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sEHqthyEGg3FQP4-vdL9zkhSJmQEaIc_2:head",
"version": "80588'26387"
},
{
"object": "6:8f1d33ec:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.wL36lNEfHGE6B4V-YT6fMuAI-n2uV30_1:head",
"version": "80588'26388"
},
{
"object": "6:8f1d33f3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.j1UloV7DpRwG0-4AKULi3CXOTC0CrdU_4:head",
"version": "80588'26389"
},
{
"object": "6:8f1d33f5:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.q3ExUB7Mp-LggBIsRBBKyIob574rNpG_3:head",
"version": "80588'26390"
},
{
"object": "6:8f1d33f9:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_72-13403:head",
"version": "80588'26391"
},
{
"object": "6:8f1d33fd:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_157-172390:head",
"version": "80588'26392"
},
{
"object": "6:8f1d3403:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Wn4ed97PvQp_oDnk_Y4xlZMke0pQ4z1_3:head",
"version": "80588'26393"
},
{
"object": "6:8f1d3405:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.uFg6eFSD3BCEqzJLCZlw546nDqqg80U_4:head",
"version": "80588'26394"
},
{
"object": "6:8f1d3436:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xPwN1sJsNFAg2Nr_Vnu4ItFgHFyu_Nx_7:head",
"version": "80588'26395"
},
{
"object": "6:8f1d3438:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Qgd9EXWAraBWnYcZ3OhFvepsT8r0Ksi_1:head",
"version": "80588'26396"
},
{
"object": "6:8f1d3464:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.1KnQBzx5u-CkXROQVUzUPfJiuTq52zH_3:head",
"version": "80588'26397"
},
{
"object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Aw2QRjM0ZXkf20MtQlhyq31hwVMEWwe_1:head",
"version": "80588'26398"
},
{
"object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xprsKnjtYu1ITOsgDtkPup9Nk6xzGTN_6:head",
"version": "80588'26399"
},
{
"object": "6:8f1d3475:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.jo4-jEQV1kB9kHewUYdbU1UUNX60EUm_1:head",
"version": "80588'26400"
},
{
"object": "6:8f1d3488:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.f9EE8lnu2FzW1_vGo9J41KY-iFpI9-f_1:head",
"version": "80588'26401"
},
{
"object": "6:8f1d3491:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_11-138415:head",
"version": "80588'26402"
},
{
"object": "6:8f1d34ac:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_33-123891:head",
"version": "80588'26403"
},
{
"object": "6:8f1d34ad:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.x3tIcVCAeRnOJmmHhJ2hxCjsUZ3GBsK_3:head",
"version": "80588'26404"
},
{
"object": "6:8f1d34bc:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.9yQELiJhCNs5Z-foJHH3vEfQ_1TZdBF_1:head",
"version": "80588'26405"
},
{
"object": "6:8f1d34d1:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Me9Xl2_j9z2_yE1laHWpHdJ2fXvwZDZ_1:head",
"version": "80588'26406"
},
{
"object": "6:8f1d34fa:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Y5VD6lU4UqjXEh5Yb9GVwusKpsqGzuG_4:head",
"version": "80588'26407"
},
{
"object": "6:8f1d3503:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_58-178651:head",
"version": "80588'26408"
},
{
"object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.FFyoywHasB0i44Y9ZtCi_YgtYhm8cP6_6:head",
"version": "80588'26409"
},
{
"object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.lHmY-Z-Q3Ewx7kHwCoLFoFU70Xtrgzu_1:head",
"version": "80588'26410"
},
{
"object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.ETPvUyLCwz5Bfmh22s_odAwEIMCC_Vn_2:head",
"version": "80588'26411"
},
{
"object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.EdYbzDvA-7HpfrM0DMfHRgalz8ZiLhJ_1:head",
"version": "80588'26412"
},
{
"object": "6:8f1d3529:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1186045.1_164-1500436:head",
"version": "80588'26413"
}
]
},
"peer_backfill_info": [
"727",
{
"begin": "MAX",
"end": "MAX",
"objects": []
}
],
"backfills_in_flight": [
"6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head"
],
"recovering": [
"6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head"
],
"pg_backend": {
"pull_from_peer": [],
"pushing": [
{
"pushing": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head",
"pushing_to": [
{
"pushing_to": "727",
"push_info": {
"recovery_progress": {
"first?": 0,
"data_complete?": 1,
"data_recovered_to": 4194304,
"omap_complete?": 1,
"omap_recovered_to": ""
},
"recovery_info": {
"object": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head",
"at_version": "80588'26366",
"size": "4194304",
"object_info": {
"oid": {
"oid": "d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605",
"key": "",
"snapid": -2,
"hash": 4182554865,
"max": 0,
"pool": 6,
"namespace": ""
},
"version": "80588'26366",
"prior_version": "6896'10987",
"last_reqid": "osd.313.0:4380013",
"user_version": 10987,
"size": 4194304,
"mtime": "2023-12-19T07:17:04.766167+0000",
"local_mtime": "2023-12-19T07:17:04.799712+0000",
"lost": 0,
"flags": [
"dirty",
"data_digest",
"omap_digest"
],
"truncate_seq": 0,
"truncate_size": 0,
"data_digest": "0x9cdfa22f",
"omap_digest": "0xffffffff",
"expected_object_size": 0,
"expected_write_size": 0,
"alloc_hint_flags": 0,
"manifest": {
"type": 0
},
"watchers": {}
},
"snapset": {
"seq": 0,
"clones": []
},
"copy_subset": "[0~4194304]",
"clone_subset": "{}",
"object_exist": "0"
}
}
}
]
}
]
}
}
},
{
"name": "Started",
"enter_time": "2024-02-03T22:31:13.882074+0000"
}
],
"scrubber": {
"active": false,
"must_scrub": false,
"must_deep_scrub": false,
"must_repair": false,
"need_auto": false,
"scrub_reg_stamp": "2024-01-30T15:39:19.044093+0000",
"schedule": "queued for deep scrub"
},
"agent_state": {}
}
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
