After wrangling with this myself, both with 17.2.7 and to an extent with 17.2.5, I'd like to follow up here and ask: Those who have experienced this, were the affected PGs * Part of an EC pool? * Part of an HDD pool? * Both? > > You don't say anything about the Ceph version you are running. > I had an similar issue with 17.2.7, and is seams to be an issue with mclock, > when I switch to wpq everything worked again. > > You can read more about it here > https://lists.ceph.io/hyperkitty/list/ceph-users@xxxxxxx/thread/IPHBE3DLW5ABCZHSNYOBUBSI3TLWVD22/#OE3QXLAJIY6NU7PNMGHP47UK2CBZJPUG > > - Kai Stian Olstad > > > On Tue, Feb 06, 2024 at 06:35:26AM -0000, LeonGao wrote: >> Hi community >> >> We have a new Ceph cluster deployment with 100 nodes. When we are draining an OSD host from the cluster, we see a small amount of PGs that cannot make any progress to the end. From the logs and metrics, it seems like the recovery progress is stuck (0 recovery ops for several days). Would like to get some ideas on this. Re-peering and OSD restart do resolve to mitigate the issue but we want to get to the root cause of it as draining and recovery happen frequently. >> >> I have put some debugging information below. Any help is appreciated, thanks! >> >> ceph -s >> pgs: 4210926/7380034104 objects misplaced (0.057%) >> 41198 active+clean >> 71 active+remapped+backfilling >> 12 active+recovering >> >> One of the stuck PG: >> 6.38f1 active+remapped+backfilling [313,643,727] 313 [313,643,717] 313 >> >> PG query result: >> >> ceph pg 6.38f1 query >> { >> "snap_trimq": "[]", >> "snap_trimq_len": 0, >> "state": "active+remapped+backfilling", >> "epoch": 246856, >> "up": [ >> 313, >> 643, >> 727 >> ], >> "acting": [ >> 313, >> 643, >> 717 >> ], >> "backfill_targets": [ >> "727" >> ], >> "acting_recovery_backfill": [ >> "313", >> "643", >> "717", >> "727" >> ], >> "info": { >> "pgid": "6.38f1", >> "last_update": "212333'38916", >> "last_complete": "212333'38916", >> "log_tail": "80608'37589", >> "last_user_version": 38833, >> "last_backfill": "MAX", >> "purged_snaps": [], >> "history": { >> "epoch_created": 3726, >> "epoch_pool_created": 3279, >> "last_epoch_started": 243987, >> "last_interval_started": 243986, >> "last_epoch_clean": 220174, >> "last_interval_clean": 220173, >> "last_epoch_split": 3726, >> "last_epoch_marked_full": 0, >> "same_up_since": 238347, >> "same_interval_since": 243986, >> "same_primary_since": 3728, >> "last_scrub": "212333'38916", >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "last_deep_scrub": "212333'38916", >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "prior_readable_until_ub": 0 >> }, >> "stats": { >> "version": "212333'38916", >> "reported_seq": 413425, >> "reported_epoch": 246856, >> "state": "active+remapped+backfilling", >> "last_fresh": "2024-02-05T21:14:40.838785+0000", >> "last_change": "2024-02-03T22:33:43.052272+0000", >> "last_active": "2024-02-05T21:14:40.838785+0000", >> "last_peered": "2024-02-05T21:14:40.838785+0000", >> "last_clean": "2024-02-03T04:26:35.168232+0000", >> "last_became_active": "2024-02-03T22:31:16.037823+0000", >> "last_became_peered": "2024-02-03T22:31:16.037823+0000", >> "last_unstale": "2024-02-05T21:14:40.838785+0000", >> "last_undegraded": "2024-02-05T21:14:40.838785+0000", >> "last_fullsized": "2024-02-05T21:14:40.838785+0000", >> "mapping_epoch": 243986, >> "log_start": "80608'37589", >> "ondisk_log_start": "80608'37589", >> "created": 3726, >> "last_epoch_clean": 220174, >> "parent": "0.0", >> "parent_split_bits": 14, >> "last_scrub": "212333'38916", >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "last_deep_scrub": "212333'38916", >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "objects_scrubbed": 17743, >> "log_size": 1327, >> "log_dups_size": 3000, >> "ondisk_log_size": 1327, >> "stats_invalid": false, >> "dirty_stats_invalid": false, >> "omap_stats_invalid": false, >> "hitset_stats_invalid": false, >> "hitset_bytes_stats_invalid": false, >> "pin_stats_invalid": false, >> "manifest_stats_invalid": false, >> "snaptrimq_len": 0, >> "last_scrub_duration": 29, >> "scrub_schedule": "queued for deep scrub", >> "scrub_duration": 28.699844475999999, >> "objects_trimmed": 0, >> "snaptrim_duration": 0, >> "stat_sum": { >> "num_bytes": 67167022618, >> "num_objects": 17743, >> "num_object_clones": 0, >> "num_object_copies": 53229, >> "num_objects_missing_on_primary": 0, >> "num_objects_missing": 0, >> "num_objects_degraded": 0, >> "num_objects_misplaced": 12401, >> "num_objects_unfound": 0, >> "num_objects_dirty": 17743, >> "num_whiteouts": 0, >> "num_read": 16640, >> "num_read_kb": 30923655, >> "num_write": 55094, >> "num_write_kb": 71824558, >> "num_scrub_errors": 0, >> "num_shallow_scrub_errors": 0, >> "num_deep_scrub_errors": 0, >> "num_objects_recovered": 40746, >> "num_bytes_recovered": 154245233300, >> "num_keys_recovered": 0, >> "num_objects_omap": 0, >> "num_objects_hit_set_archive": 0, >> "num_bytes_hit_set_archive": 0, >> "num_flush": 0, >> "num_flush_kb": 0, >> "num_evict": 0, >> "num_evict_kb": 0, >> "num_promote": 0, >> "num_flush_mode_high": 0, >> "num_flush_mode_low": 0, >> "num_evict_mode_some": 0, >> "num_evict_mode_full": 0, >> "num_objects_pinned": 0, >> "num_legacy_snapsets": 0, >> "num_large_omap_objects": 0, >> "num_objects_manifest": 0, >> "num_omap_bytes": 0, >> "num_omap_keys": 0, >> "num_objects_repaired": 0 >> }, >> "up": [ >> 313, >> 643, >> 727 >> ], >> "acting": [ >> 313, >> 643, >> 717 >> ], >> "avail_no_missing": [ >> "313", >> "643", >> "717" >> ], >> "object_location_counts": [ >> { >> "shards": "313,643,717", >> "objects": 17743 >> } >> ], >> "blocked_by": [], >> "up_primary": 313, >> "acting_primary": 313, >> "purged_snaps": [] >> }, >> "empty": 0, >> "dne": 0, >> "incomplete": 0, >> "last_epoch_started": 243987, >> "hit_set_history": { >> "current_last_update": "0'0", >> "history": [] >> } >> }, >> "peer_info": [ >> { >> "peer": "643", >> "pgid": "6.38f1", >> "last_update": "212333'38916", >> "last_complete": "212333'38916", >> "log_tail": "80608'37589", >> "last_user_version": 38833, >> "last_backfill": "MAX", >> "purged_snaps": [], >> "history": { >> "epoch_created": 3726, >> "epoch_pool_created": 3279, >> "last_epoch_started": 243987, >> "last_interval_started": 243986, >> "last_epoch_clean": 220174, >> "last_interval_clean": 220173, >> "last_epoch_split": 3726, >> "last_epoch_marked_full": 0, >> "same_up_since": 238347, >> "same_interval_since": 243986, >> "same_primary_since": 3728, >> "last_scrub": "212333'38916", >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "last_deep_scrub": "212333'38916", >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "prior_readable_until_ub": 0 >> }, >> "stats": { >> "version": "212331'38915", >> "reported_seq": 362849, >> "reported_epoch": 212333, >> "state": "active+clean+scrubbing+deep", >> "last_fresh": "2024-01-28T07:43:40.515729+0000", >> "last_change": "2024-01-28T06:20:14.969313+0000", >> "last_active": "2024-01-28T07:43:40.515729+0000", >> "last_peered": "2024-01-28T07:43:40.515729+0000", >> "last_clean": "2024-01-28T07:43:40.515729+0000", >> "last_became_active": "2024-01-23T10:01:07.710478+0000", >> "last_became_peered": "2024-01-23T10:01:07.710478+0000", >> "last_unstale": "2024-01-28T07:43:40.515729+0000", >> "last_undegraded": "2024-01-28T07:43:40.515729+0000", >> "last_fullsized": "2024-01-28T07:43:40.515729+0000", >> "mapping_epoch": 243986, >> "log_start": "80608'37589", >> "ondisk_log_start": "80608'37589", >> "created": 3726, >> "last_epoch_clean": 123203, >> "parent": "0.0", >> "parent_split_bits": 14, >> "last_scrub": "80609'38697", >> "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", >> "last_deep_scrub": "80609'38697", >> "last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000", >> "last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000", >> "objects_scrubbed": 17724, >> "log_size": 1326, >> "log_dups_size": 3000, >> "ondisk_log_size": 1326, >> "stats_invalid": false, >> "dirty_stats_invalid": false, >> "omap_stats_invalid": false, >> "hitset_stats_invalid": false, >> "hitset_bytes_stats_invalid": false, >> "pin_stats_invalid": false, >> "manifest_stats_invalid": false, >> "snaptrimq_len": 0, >> "last_scrub_duration": 29, >> "scrub_schedule": "deep scrubbing for 5005s", >> "scrub_duration": 28.446739284, >> "objects_trimmed": 0, >> "snaptrim_duration": 0, >> "stat_sum": { >> "num_bytes": 67167022618, >> "num_objects": 17743, >> "num_object_clones": 0, >> "num_object_copies": 53229, >> "num_objects_missing_on_primary": 0, >> "num_objects_missing": 0, >> "num_objects_degraded": 0, >> "num_objects_misplaced": 0, >> "num_objects_unfound": 0, >> "num_objects_dirty": 17743, >> "num_whiteouts": 0, >> "num_read": 16640, >> "num_read_kb": 30923655, >> "num_write": 55094, >> "num_write_kb": 71824558, >> "num_scrub_errors": 0, >> "num_shallow_scrub_errors": 0, >> "num_deep_scrub_errors": 0, >> "num_objects_recovered": 35403, >> "num_bytes_recovered": 134021819371, >> "num_keys_recovered": 0, >> "num_objects_omap": 0, >> "num_objects_hit_set_archive": 0, >> "num_bytes_hit_set_archive": 0, >> "num_flush": 0, >> "num_flush_kb": 0, >> "num_evict": 0, >> "num_evict_kb": 0, >> "num_promote": 0, >> "num_flush_mode_high": 0, >> "num_flush_mode_low": 0, >> "num_evict_mode_some": 0, >> "num_evict_mode_full": 0, >> "num_objects_pinned": 0, >> "num_legacy_snapsets": 0, >> "num_large_omap_objects": 0, >> "num_objects_manifest": 0, >> "num_omap_bytes": 0, >> "num_omap_keys": 0, >> "num_objects_repaired": 0 >> }, >> "up": [ >> 313, >> 643, >> 727 >> ], >> "acting": [ >> 313, >> 643, >> 717 >> ], >> "avail_no_missing": [], >> "object_location_counts": [], >> "blocked_by": [], >> "up_primary": 313, >> "acting_primary": 313, >> "purged_snaps": [] >> }, >> "empty": 0, >> "dne": 0, >> "incomplete": 0, >> "last_epoch_started": 243987, >> "hit_set_history": { >> "current_last_update": "0'0", >> "history": [] >> } >> }, >> { >> "peer": "717", >> "pgid": "6.38f1", >> "last_update": "212333'38916", >> "last_complete": "212333'38916", >> "log_tail": "80608'37589", >> "last_user_version": 0, >> "last_backfill": "MAX", >> "purged_snaps": [], >> "history": { >> "epoch_created": 3726, >> "epoch_pool_created": 3279, >> "last_epoch_started": 243987, >> "last_interval_started": 243986, >> "last_epoch_clean": 220174, >> "last_interval_clean": 220173, >> "last_epoch_split": 3726, >> "last_epoch_marked_full": 0, >> "same_up_since": 238347, >> "same_interval_since": 243986, >> "same_primary_since": 3728, >> "last_scrub": "212333'38916", >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "last_deep_scrub": "212333'38916", >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "prior_readable_until_ub": 0 >> }, >> "stats": { >> "version": "212331'38915", >> "reported_seq": 362849, >> "reported_epoch": 212333, >> "state": "active+clean+scrubbing+deep", >> "last_fresh": "2024-01-28T07:43:40.515729+0000", >> "last_change": "2024-01-28T06:20:14.969313+0000", >> "last_active": "2024-01-28T07:43:40.515729+0000", >> "last_peered": "2024-01-28T07:43:40.515729+0000", >> "last_clean": "2024-01-28T07:43:40.515729+0000", >> "last_became_active": "2024-01-23T10:01:07.710478+0000", >> "last_became_peered": "2024-01-23T10:01:07.710478+0000", >> "last_unstale": "2024-01-28T07:43:40.515729+0000", >> "last_undegraded": "2024-01-28T07:43:40.515729+0000", >> "last_fullsized": "2024-01-28T07:43:40.515729+0000", >> "mapping_epoch": 243986, >> "log_start": "80608'37589", >> "ondisk_log_start": "80608'37589", >> "created": 3726, >> "last_epoch_clean": 123203, >> "parent": "0.0", >> "parent_split_bits": 14, >> "last_scrub": "80609'38697", >> "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", >> "last_deep_scrub": "80609'38697", >> "last_deep_scrub_stamp": "2024-01-07T08:20:59.246940+0000", >> "last_clean_scrub_stamp": "2024-01-09T20:27:40.795531+0000", >> "objects_scrubbed": 17724, >> "log_size": 1326, >> "log_dups_size": 3000, >> "ondisk_log_size": 1326, >> "stats_invalid": false, >> "dirty_stats_invalid": false, >> "omap_stats_invalid": false, >> "hitset_stats_invalid": false, >> "hitset_bytes_stats_invalid": false, >> "pin_stats_invalid": false, >> "manifest_stats_invalid": false, >> "snaptrimq_len": 0, >> "last_scrub_duration": 29, >> "scrub_schedule": "deep scrubbing for 5005s", >> "scrub_duration": 28.446739284, >> "objects_trimmed": 0, >> "snaptrim_duration": 0, >> "stat_sum": { >> "num_bytes": 67167022618, >> "num_objects": 17743, >> "num_object_clones": 0, >> "num_object_copies": 53229, >> "num_objects_missing_on_primary": 0, >> "num_objects_missing": 0, >> "num_objects_degraded": 0, >> "num_objects_misplaced": 0, >> "num_objects_unfound": 0, >> "num_objects_dirty": 17743, >> "num_whiteouts": 0, >> "num_read": 16640, >> "num_read_kb": 30923655, >> "num_write": 55094, >> "num_write_kb": 71824558, >> "num_scrub_errors": 0, >> "num_shallow_scrub_errors": 0, >> "num_deep_scrub_errors": 0, >> "num_objects_recovered": 35403, >> "num_bytes_recovered": 134021819371, >> "num_keys_recovered": 0, >> "num_objects_omap": 0, >> "num_objects_hit_set_archive": 0, >> "num_bytes_hit_set_archive": 0, >> "num_flush": 0, >> "num_flush_kb": 0, >> "num_evict": 0, >> "num_evict_kb": 0, >> "num_promote": 0, >> "num_flush_mode_high": 0, >> "num_flush_mode_low": 0, >> "num_evict_mode_some": 0, >> "num_evict_mode_full": 0, >> "num_objects_pinned": 0, >> "num_legacy_snapsets": 0, >> "num_large_omap_objects": 0, >> "num_objects_manifest": 0, >> "num_omap_bytes": 0, >> "num_omap_keys": 0, >> "num_objects_repaired": 0 >> }, >> "up": [ >> 313, >> 643, >> 727 >> ], >> "acting": [ >> 313, >> 643, >> 717 >> ], >> "avail_no_missing": [], >> "object_location_counts": [], >> "blocked_by": [], >> "up_primary": 313, >> "acting_primary": 313, >> "purged_snaps": [] >> }, >> "empty": 0, >> "dne": 0, >> "incomplete": 0, >> "last_epoch_started": 243987, >> "hit_set_history": { >> "current_last_update": "0'0", >> "history": [] >> } >> }, >> { >> "peer": "726", >> "pgid": "6.38f1", >> "last_update": "212333'38916", >> "last_complete": "212333'38916", >> "log_tail": "80608'37589", >> "last_user_version": 0, >> "last_backfill": "MIN", >> "purged_snaps": [], >> "history": { >> "epoch_created": 3726, >> "epoch_pool_created": 3279, >> "last_epoch_started": 238348, >> "last_interval_started": 238347, >> "last_epoch_clean": 220174, >> "last_interval_clean": 220173, >> "last_epoch_split": 3726, >> "last_epoch_marked_full": 0, >> "same_up_since": 238347, >> "same_interval_since": 243986, >> "same_primary_since": 3728, >> "last_scrub": "212333'38916", >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "last_deep_scrub": "212333'38916", >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "prior_readable_until_ub": 11.834917336 >> }, >> "stats": { >> "version": "0'0", >> "reported_seq": 0, >> "reported_epoch": 0, >> "state": "unknown", >> "last_fresh": "0.000000", >> "last_change": "0.000000", >> "last_active": "0.000000", >> "last_peered": "0.000000", >> "last_clean": "0.000000", >> "last_became_active": "0.000000", >> "last_became_peered": "0.000000", >> "last_unstale": "0.000000", >> "last_undegraded": "0.000000", >> "last_fullsized": "0.000000", >> "mapping_epoch": 243986, >> "log_start": "0'0", >> "ondisk_log_start": "0'0", >> "created": 0, >> "last_epoch_clean": 0, >> "parent": "0.0", >> "parent_split_bits": 0, >> "last_scrub": "0'0", >> "last_scrub_stamp": "0.000000", >> "last_deep_scrub": "0'0", >> "last_deep_scrub_stamp": "0.000000", >> "last_clean_scrub_stamp": "0.000000", >> "objects_scrubbed": 0, >> "log_size": 0, >> "log_dups_size": 0, >> "ondisk_log_size": 0, >> "stats_invalid": false, >> "dirty_stats_invalid": false, >> "omap_stats_invalid": false, >> "hitset_stats_invalid": false, >> "hitset_bytes_stats_invalid": false, >> "pin_stats_invalid": false, >> "manifest_stats_invalid": false, >> "snaptrimq_len": 0, >> "last_scrub_duration": 0, >> "scrub_schedule": "--", >> "scrub_duration": 0, >> "objects_trimmed": 0, >> "snaptrim_duration": 0, >> "stat_sum": { >> "num_bytes": 4194304, >> "num_objects": 0, >> "num_object_clones": 0, >> "num_object_copies": 0, >> "num_objects_missing_on_primary": 0, >> "num_objects_missing": 17743, >> "num_objects_degraded": 0, >> "num_objects_misplaced": 0, >> "num_objects_unfound": 0, >> "num_objects_dirty": 0, >> "num_whiteouts": 0, >> "num_read": 0, >> "num_read_kb": 0, >> "num_write": 0, >> "num_write_kb": 0, >> "num_scrub_errors": 0, >> "num_shallow_scrub_errors": 0, >> "num_deep_scrub_errors": 0, >> "num_objects_recovered": 0, >> "num_bytes_recovered": 0, >> "num_keys_recovered": 0, >> "num_objects_omap": 0, >> "num_objects_hit_set_archive": 0, >> "num_bytes_hit_set_archive": 0, >> "num_flush": 0, >> "num_flush_kb": 0, >> "num_evict": 0, >> "num_evict_kb": 0, >> "num_promote": 0, >> "num_flush_mode_high": 0, >> "num_flush_mode_low": 0, >> "num_evict_mode_some": 0, >> "num_evict_mode_full": 0, >> "num_objects_pinned": 0, >> "num_legacy_snapsets": 0, >> "num_large_omap_objects": 0, >> "num_objects_manifest": 0, >> "num_omap_bytes": 0, >> "num_omap_keys": 0, >> "num_objects_repaired": 0 >> }, >> "up": [ >> 313, >> 643, >> 727 >> ], >> "acting": [ >> 313, >> 643, >> 717 >> ], >> "avail_no_missing": [], >> "object_location_counts": [], >> "blocked_by": [], >> "up_primary": 313, >> "acting_primary": 313, >> "purged_snaps": [] >> }, >> "empty": 0, >> "dne": 0, >> "incomplete": 1, >> "last_epoch_started": 238320, >> "hit_set_history": { >> "current_last_update": "0'0", >> "history": [] >> } >> }, >> { >> "peer": "727", >> "pgid": "6.38f1", >> "last_update": "212333'38916", >> "last_complete": "212333'38916", >> "log_tail": "80608'37589", >> "last_user_version": 0, >> "last_backfill": "6:8f1d3293:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.V-JlvDFjbQNgn8GtBLN2LePecv5fLKf_3:head", >> "purged_snaps": [], >> "history": { >> "epoch_created": 3726, >> "epoch_pool_created": 3279, >> "last_epoch_started": 243987, >> "last_interval_started": 243986, >> "last_epoch_clean": 220174, >> "last_interval_clean": 220173, >> "last_epoch_split": 3726, >> "last_epoch_marked_full": 0, >> "same_up_since": 238347, >> "same_interval_since": 243986, >> "same_primary_since": 3728, >> "last_scrub": "212333'38916", >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "last_deep_scrub": "212333'38916", >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", >> "prior_readable_until_ub": 0 >> }, >> "stats": { >> "version": "0'0", >> "reported_seq": 0, >> "reported_epoch": 0, >> "state": "unknown", >> "last_fresh": "0.000000", >> "last_change": "0.000000", >> "last_active": "0.000000", >> "last_peered": "0.000000", >> "last_clean": "0.000000", >> "last_became_active": "0.000000", >> "last_became_peered": "0.000000", >> "last_unstale": "0.000000", >> "last_undegraded": "0.000000", >> "last_fullsized": "0.000000", >> "mapping_epoch": 243986, >> "log_start": "0'0", >> "ondisk_log_start": "0'0", >> "created": 0, >> "last_epoch_clean": 0, >> "parent": "0.0", >> "parent_split_bits": 0, >> "last_scrub": "0'0", >> "last_scrub_stamp": "0.000000", >> "last_deep_scrub": "0'0", >> "last_deep_scrub_stamp": "0.000000", >> "last_clean_scrub_stamp": "0.000000", >> "objects_scrubbed": 0, >> "log_size": 0, >> "log_dups_size": 0, >> "ondisk_log_size": 0, >> "stats_invalid": false, >> "dirty_stats_invalid": false, >> "omap_stats_invalid": false, >> "hitset_stats_invalid": false, >> "hitset_bytes_stats_invalid": false, >> "pin_stats_invalid": false, >> "manifest_stats_invalid": false, >> "snaptrimq_len": 0, >> "last_scrub_duration": 0, >> "scrub_schedule": "--", >> "scrub_duration": 0, >> "objects_trimmed": 0, >> "snaptrim_duration": 0, >> "stat_sum": { >> "num_bytes": 20227608233, >> "num_objects": 5342, >> "num_object_clones": 0, >> "num_object_copies": 0, >> "num_objects_missing_on_primary": 0, >> "num_objects_missing": 12401, >> "num_objects_degraded": 0, >> "num_objects_misplaced": 0, >> "num_objects_unfound": 0, >> "num_objects_dirty": 5342, >> "num_whiteouts": 0, >> "num_read": 0, >> "num_read_kb": 0, >> "num_write": 0, >> "num_write_kb": 0, >> "num_scrub_errors": 0, >> "num_shallow_scrub_errors": 0, >> "num_deep_scrub_errors": 0, >> "num_objects_recovered": 0, >> "num_bytes_recovered": 0, >> "num_keys_recovered": 0, >> "num_objects_omap": 0, >> "num_objects_hit_set_archive": 0, >> "num_bytes_hit_set_archive": 0, >> "num_flush": 0, >> "num_flush_kb": 0, >> "num_evict": 0, >> "num_evict_kb": 0, >> "num_promote": 0, >> "num_flush_mode_high": 0, >> "num_flush_mode_low": 0, >> "num_evict_mode_some": 0, >> "num_evict_mode_full": 0, >> "num_objects_pinned": 0, >> "num_legacy_snapsets": 0, >> "num_large_omap_objects": 0, >> "num_objects_manifest": 0, >> "num_omap_bytes": 0, >> "num_omap_keys": 0, >> "num_objects_repaired": 0 >> }, >> "up": [ >> 313, >> 643, >> 727 >> ], >> "acting": [ >> 313, >> 643, >> 717 >> ], >> "avail_no_missing": [], >> "object_location_counts": [], >> "blocked_by": [], >> "up_primary": 313, >> "acting_primary": 313, >> "purged_snaps": [] >> }, >> "empty": 0, >> "dne": 0, >> "incomplete": 1, >> "last_epoch_started": 243987, >> "hit_set_history": { >> "current_last_update": "0'0", >> "history": [] >> } >> } >> ], >> "recovery_state": [ >> { >> "name": "Started/Primary/Active", >> "enter_time": "2024-02-03T22:31:15.023323+0000", >> "might_have_unfound": [], >> "recovery_progress": { >> "backfill_targets": [ >> "727" >> ], >> "waiting_on_backfill": [], >> "last_backfill_started": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", >> "backfill_info": { >> "begin": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", >> "end": "6:8f1d3549:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.PTxEYjlIXk2vkWpRgKUOu4ygweU-Jqy_5:head", >> "objects": [ >> { >> "object": "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", >> "version": "80588'26367" >> }, >> { >> "object": "6:8f1d32b0:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_18-71435:head", >> "version": "80588'26368" >> }, >> { >> "object": "6:8f1d32c2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.o_s0NGR7mrH-4JKpvYS6Ip14hbm4w5o_5:head", >> "version": "80588'26369" >> }, >> { >> "object": "6:8f1d32c3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WV1Ja5V21BKfmI5e1W90a7iQXdcTnSG_6:head", >> "version": "80588'26370" >> }, >> { >> "object": "6:8f1d32c4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.96tkpnCzKmtnPTLewpai2cXSr_z_QcR_1:head", >> "version": "80588'26371" >> }, >> { >> "object": "6:8f1d331d:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.InACTSlwV7irTe0g9cLnadT8EbQr4Cl_5:head", >> "version": "80588'26372" >> }, >> { >> "object": "6:8f1d3326:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_233-155987:head", >> "version": "80588'26373" >> }, >> { >> "object": "6:8f1d3332:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.5aipEOj0vV3h4jdSf6tkAUn64yJmvoH_1:head", >> "version": "80588'26374" >> }, >> { >> "object": "6:8f1d3343:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QTzZJhDV6JLC5wJ5BDxHvK4sfp3Hr6o_6:head", >> "version": "80588'26375" >> }, >> { >> "object": "6:8f1d334b:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QgMCmHWV87HlAjqkfvbO9X5OUZrgIPc_2:head", >> "version": "80588'26376" >> }, >> { >> "object": "6:8f1d3355:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.H4j6DWgQa0jsrl-xzgugsW22GHCOgGL_7:head", >> "version": "80588'26377" >> }, >> { >> "object": "6:8f1d3379:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.25g7c0w-VnGqX-z_m53YmnoWUO0R56N_5:head", >> "version": "80588'26378" >> }, >> { >> "object": "6:8f1d337c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WMmV4BoLMm4q6BztoZ1uPc4boiJ-zhW_1:head", >> "version": "80588'26379" >> }, >> { >> "object": "6:8f1d3385:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_184-108817:head", >> "version": "80588'26380" >> }, >> { >> "object": "6:8f1d3389:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sGNiGV9K-TGyTYRqWSOcSIXkBBOT93X_3:head", >> "version": "80588'26381" >> }, >> { >> "object": "6:8f1d339c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.NkMm_dhUxRyxXhBDYqGCuOaSvwNvSuJ_3:head", >> "version": "80588'26382" >> }, >> { >> "object": "6:8f1d33a3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.BV__z8jIWqcjMsTf40MvaARU1dVtCv2_4:head", >> "version": "80588'26383" >> }, >> { >> "object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.XJLSyRSnrDupnOjWzHKc4wykPJjqR5g_1:head", >> "version": "80588'26384" >> }, >> { >> "object": "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.oYSnMakOIpFmfLdym2oCx-fLBDCLDmr_4:head", >> "version": "80588'26385" >> }, >> { >> "object": "6:8f1d33b4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_135-2779:head", >> "version": "80588'26386" >> }, >> { >> "object": "6:8f1d33b6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sEHqthyEGg3FQP4-vdL9zkhSJmQEaIc_2:head", >> "version": "80588'26387" >> }, >> { >> "object": "6:8f1d33ec:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.wL36lNEfHGE6B4V-YT6fMuAI-n2uV30_1:head", >> "version": "80588'26388" >> }, >> { >> "object": "6:8f1d33f3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.j1UloV7DpRwG0-4AKULi3CXOTC0CrdU_4:head", >> "version": "80588'26389" >> }, >> { >> "object": "6:8f1d33f5:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.q3ExUB7Mp-LggBIsRBBKyIob574rNpG_3:head", >> "version": "80588'26390" >> }, >> { >> "object": "6:8f1d33f9:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_72-13403:head", >> "version": "80588'26391" >> }, >> { >> "object": "6:8f1d33fd:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_157-172390:head", >> "version": "80588'26392" >> }, >> { >> "object": "6:8f1d3403:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Wn4ed97PvQp_oDnk_Y4xlZMke0pQ4z1_3:head", >> "version": "80588'26393" >> }, >> { >> "object": "6:8f1d3405:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.uFg6eFSD3BCEqzJLCZlw546nDqqg80U_4:head", >> "version": "80588'26394" >> }, >> { >> "object": "6:8f1d3436:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xPwN1sJsNFAg2Nr_Vnu4ItFgHFyu_Nx_7:head", >> "version": "80588'26395" >> }, >> { >> "object": "6:8f1d3438:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Qgd9EXWAraBWnYcZ3OhFvepsT8r0Ksi_1:head", >> "version": "80588'26396" >> }, >> { >> "object": "6:8f1d3464:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.1KnQBzx5u-CkXROQVUzUPfJiuTq52zH_3:head", >> "version": "80588'26397" >> }, >> { >> "object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Aw2QRjM0ZXkf20MtQlhyq31hwVMEWwe_1:head", >> "version": "80588'26398" >> }, >> { >> "object": "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xprsKnjtYu1ITOsgDtkPup9Nk6xzGTN_6:head", >> "version": "80588'26399" >> }, >> { >> "object": "6:8f1d3475:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.jo4-jEQV1kB9kHewUYdbU1UUNX60EUm_1:head", >> "version": "80588'26400" >> }, >> { >> "object": "6:8f1d3488:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.f9EE8lnu2FzW1_vGo9J41KY-iFpI9-f_1:head", >> "version": "80588'26401" >> }, >> { >> "object": "6:8f1d3491:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_11-138415:head", >> "version": "80588'26402" >> }, >> { >> "object": "6:8f1d34ac:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_33-123891:head", >> "version": "80588'26403" >> }, >> { >> "object": "6:8f1d34ad:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.x3tIcVCAeRnOJmmHhJ2hxCjsUZ3GBsK_3:head", >> "version": "80588'26404" >> }, >> { >> "object": "6:8f1d34bc:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.9yQELiJhCNs5Z-foJHH3vEfQ_1TZdBF_1:head", >> "version": "80588'26405" >> }, >> { >> "object": "6:8f1d34d1:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Me9Xl2_j9z2_yE1laHWpHdJ2fXvwZDZ_1:head", >> "version": "80588'26406" >> }, >> { >> "object": "6:8f1d34fa:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Y5VD6lU4UqjXEh5Yb9GVwusKpsqGzuG_4:head", >> "version": "80588'26407" >> }, >> { >> "object": "6:8f1d3503:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_58-178651:head", >> "version": "80588'26408" >> }, >> { >> "object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.FFyoywHasB0i44Y9ZtCi_YgtYhm8cP6_6:head", >> "version": "80588'26409" >> }, >> { >> "object": "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.lHmY-Z-Q3Ewx7kHwCoLFoFU70Xtrgzu_1:head", >> "version": "80588'26410" >> }, >> { >> "object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.ETPvUyLCwz5Bfmh22s_odAwEIMCC_Vn_2:head", >> "version": "80588'26411" >> }, >> { >> "object": "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.EdYbzDvA-7HpfrM0DMfHRgalz8ZiLhJ_1:head", >> "version": "80588'26412" >> }, >> { >> "object": "6:8f1d3529:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1186045.1_164-1500436:head", >> "version": "80588'26413" >> } >> ] >> }, >> "peer_backfill_info": [ >> "727", >> { >> "begin": "MAX", >> "end": "MAX", >> "objects": [] >> } >> ], >> "backfills_in_flight": [ >> "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" >> ], >> "recovering": [ >> "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" >> ], >> "pg_backend": { >> "pull_from_peer": [], >> "pushing": [ >> { >> "pushing": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", >> "pushing_to": [ >> { >> "pushing_to": "727", >> "push_info": { >> "recovery_progress": { >> "first?": 0, >> "data_complete?": 1, >> "data_recovered_to": 4194304, >> "omap_complete?": 1, >> "omap_recovered_to": "" >> }, >> "recovery_info": { >> "object": "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", >> "at_version": "80588'26366", >> "size": "4194304", >> "object_info": { >> "oid": { >> "oid": "d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605", >> "key": "", >> "snapid": -2, >> "hash": 4182554865, >> "max": 0, >> "pool": 6, >> "namespace": "" >> }, >> "version": "80588'26366", >> "prior_version": "6896'10987", >> "last_reqid": "osd.313.0:4380013", >> "user_version": 10987, >> "size": 4194304, >> "mtime": "2023-12-19T07:17:04.766167+0000", >> "local_mtime": "2023-12-19T07:17:04.799712+0000", >> "lost": 0, >> "flags": [ >> "dirty", >> "data_digest", >> "omap_digest" >> ], >> "truncate_seq": 0, >> "truncate_size": 0, >> "data_digest": "0x9cdfa22f", >> "omap_digest": "0xffffffff", >> "expected_object_size": 0, >> "expected_write_size": 0, >> "alloc_hint_flags": 0, >> "manifest": { >> "type": 0 >> }, >> "watchers": {} >> }, >> "snapset": { >> "seq": 0, >> "clones": [] >> }, >> "copy_subset": "[0~4194304]", >> "clone_subset": "{}", >> "object_exist": "0" >> } >> } >> } >> ] >> } >> ] >> } >> } >> }, >> { >> "name": "Started", >> "enter_time": "2024-02-03T22:31:13.882074+0000" >> } >> ], >> "scrubber": { >> "active": false, >> "must_scrub": false, >> "must_deep_scrub": false, >> "must_repair": false, >> "need_auto": false, >> "scrub_reg_stamp": "2024-01-30T15:39:19.044093+0000", >> "schedule": "queued for deep scrub" >> }, >> "agent_state": {} >> } >> _______________________________________________ >> ceph-users mailing list -- ceph-users@xxxxxxx >> To unsubscribe send an email to ceph-users-leave@xxxxxxx >> > _______________________________________________ > ceph-users mailing list -- ceph-users@xxxxxxx > To unsubscribe send an email to ceph-users-leave@xxxxxxx _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |