Thanks a lot! Yes it turns out to be the same issue that you pointed to. Switching to wpq solved the issue. We are running 18.2.0. Leon On Wed, Feb 7, 2024 at 12:48 PM Kai Stian Olstad <ceph+list@xxxxxxxxxx> wrote: > You don't say anything about the Ceph version you are running. > I had an similar issue with 17.2.7, and is seams to be an issue with > mclock, > when I switch to wpq everything worked again. > > You can read more about it here > > https://lists.ceph.io/hyperkitty/list/ceph-users@xxxxxxx/thread/IPHBE3DLW5ABCZHSNYOBUBSI3TLWVD22/#OE3QXLAJIY6NU7PNMGHP47UK2CBZJPUG > > - > Kai Stian Olstad > > > On Tue, Feb 06, 2024 at 06:35:26AM -0000, LeonGao wrote: > >Hi community > > > >We have a new Ceph cluster deployment with 100 nodes. When we are > draining an OSD host from the cluster, we see a small amount of PGs that > cannot make any progress to the end. From the logs and metrics, it seems > like the recovery progress is stuck (0 recovery ops for several days). > Would like to get some ideas on this. Re-peering and OSD restart do resolve > to mitigate the issue but we want to get to the root cause of it as > draining and recovery happen frequently. > > > >I have put some debugging information below. Any help is appreciated, > thanks! > > > >ceph -s > > pgs: 4210926/7380034104 objects misplaced (0.057%) > > 41198 active+clean > > 71 active+remapped+backfilling > > 12 active+recovering > > > >One of the stuck PG: > >6.38f1 active+remapped+backfilling [313,643,727] > 313 [313,643,717] 313 > > > >PG query result: > > > >ceph pg 6.38f1 query > >{ > > "snap_trimq": "[]", > > "snap_trimq_len": 0, > > "state": "active+remapped+backfilling", > > "epoch": 246856, > > "up": [ > > 313, > > 643, > > 727 > > ], > > "acting": [ > > 313, > > 643, > > 717 > > ], > > "backfill_targets": [ > > "727" > > ], > > "acting_recovery_backfill": [ > > "313", > > "643", > > "717", > > "727" > > ], > > "info": { > > "pgid": "6.38f1", > > "last_update": "212333'38916", > > "last_complete": "212333'38916", > > "log_tail": "80608'37589", > > "last_user_version": 38833, > > "last_backfill": "MAX", > > "purged_snaps": [], > > "history": { > > "epoch_created": 3726, > > "epoch_pool_created": 3279, > > "last_epoch_started": 243987, > > "last_interval_started": 243986, > > "last_epoch_clean": 220174, > > "last_interval_clean": 220173, > > "last_epoch_split": 3726, > > "last_epoch_marked_full": 0, > > "same_up_since": 238347, > > "same_interval_since": 243986, > > "same_primary_since": 3728, > > "last_scrub": "212333'38916", > > "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "last_deep_scrub": "212333'38916", > > "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", > > "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "prior_readable_until_ub": 0 > > }, > > "stats": { > > "version": "212333'38916", > > "reported_seq": 413425, > > "reported_epoch": 246856, > > "state": "active+remapped+backfilling", > > "last_fresh": "2024-02-05T21:14:40.838785+0000", > > "last_change": "2024-02-03T22:33:43.052272+0000", > > "last_active": "2024-02-05T21:14:40.838785+0000", > > "last_peered": "2024-02-05T21:14:40.838785+0000", > > "last_clean": "2024-02-03T04:26:35.168232+0000", > > "last_became_active": "2024-02-03T22:31:16.037823+0000", > > "last_became_peered": "2024-02-03T22:31:16.037823+0000", > > "last_unstale": "2024-02-05T21:14:40.838785+0000", > > "last_undegraded": "2024-02-05T21:14:40.838785+0000", > > "last_fullsized": "2024-02-05T21:14:40.838785+0000", > > "mapping_epoch": 243986, > > "log_start": "80608'37589", > > "ondisk_log_start": "80608'37589", > > "created": 3726, > > "last_epoch_clean": 220174, > > "parent": "0.0", > > "parent_split_bits": 14, > > "last_scrub": "212333'38916", > > "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "last_deep_scrub": "212333'38916", > > "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", > > "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "objects_scrubbed": 17743, > > "log_size": 1327, > > "log_dups_size": 3000, > > "ondisk_log_size": 1327, > > "stats_invalid": false, > > "dirty_stats_invalid": false, > > "omap_stats_invalid": false, > > "hitset_stats_invalid": false, > > "hitset_bytes_stats_invalid": false, > > "pin_stats_invalid": false, > > "manifest_stats_invalid": false, > > "snaptrimq_len": 0, > > "last_scrub_duration": 29, > > "scrub_schedule": "queued for deep scrub", > > "scrub_duration": 28.699844475999999, > > "objects_trimmed": 0, > > "snaptrim_duration": 0, > > "stat_sum": { > > "num_bytes": 67167022618, > > "num_objects": 17743, > > "num_object_clones": 0, > > "num_object_copies": 53229, > > "num_objects_missing_on_primary": 0, > > "num_objects_missing": 0, > > "num_objects_degraded": 0, > > "num_objects_misplaced": 12401, > > "num_objects_unfound": 0, > > "num_objects_dirty": 17743, > > "num_whiteouts": 0, > > "num_read": 16640, > > "num_read_kb": 30923655, > > "num_write": 55094, > > "num_write_kb": 71824558, > > "num_scrub_errors": 0, > > "num_shallow_scrub_errors": 0, > > "num_deep_scrub_errors": 0, > > "num_objects_recovered": 40746, > > "num_bytes_recovered": 154245233300, > > "num_keys_recovered": 0, > > "num_objects_omap": 0, > > "num_objects_hit_set_archive": 0, > > "num_bytes_hit_set_archive": 0, > > "num_flush": 0, > > "num_flush_kb": 0, > > "num_evict": 0, > > "num_evict_kb": 0, > > "num_promote": 0, > > "num_flush_mode_high": 0, > > "num_flush_mode_low": 0, > > "num_evict_mode_some": 0, > > "num_evict_mode_full": 0, > > "num_objects_pinned": 0, > > "num_legacy_snapsets": 0, > > "num_large_omap_objects": 0, > > "num_objects_manifest": 0, > > "num_omap_bytes": 0, > > "num_omap_keys": 0, > > "num_objects_repaired": 0 > > }, > > "up": [ > > 313, > > 643, > > 727 > > ], > > "acting": [ > > 313, > > 643, > > 717 > > ], > > "avail_no_missing": [ > > "313", > > "643", > > "717" > > ], > > "object_location_counts": [ > > { > > "shards": "313,643,717", > > "objects": 17743 > > } > > ], > > "blocked_by": [], > > "up_primary": 313, > > "acting_primary": 313, > > "purged_snaps": [] > > }, > > "empty": 0, > > "dne": 0, > > "incomplete": 0, > > "last_epoch_started": 243987, > > "hit_set_history": { > > "current_last_update": "0'0", > > "history": [] > > } > > }, > > "peer_info": [ > > { > > "peer": "643", > > "pgid": "6.38f1", > > "last_update": "212333'38916", > > "last_complete": "212333'38916", > > "log_tail": "80608'37589", > > "last_user_version": 38833, > > "last_backfill": "MAX", > > "purged_snaps": [], > > "history": { > > "epoch_created": 3726, > > "epoch_pool_created": 3279, > > "last_epoch_started": 243987, > > "last_interval_started": 243986, > > "last_epoch_clean": 220174, > > "last_interval_clean": 220173, > > "last_epoch_split": 3726, > > "last_epoch_marked_full": 0, > > "same_up_since": 238347, > > "same_interval_since": 243986, > > "same_primary_since": 3728, > > "last_scrub": "212333'38916", > > "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "last_deep_scrub": "212333'38916", > > "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > > "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > > "prior_readable_until_ub": 0 > > }, > > "stats": { > > "version": "212331'38915", > > "reported_seq": 362849, > > "reported_epoch": 212333, > > "state": "active+clean+scrubbing+deep", > > "last_fresh": "2024-01-28T07:43:40.515729+0000", > > "last_change": "2024-01-28T06:20:14.969313+0000", > > "last_active": "2024-01-28T07:43:40.515729+0000", > > "last_peered": "2024-01-28T07:43:40.515729+0000", > > "last_clean": "2024-01-28T07:43:40.515729+0000", > > "last_became_active": "2024-01-23T10:01:07.710478+0000", > > "last_became_peered": "2024-01-23T10:01:07.710478+0000", > > "last_unstale": "2024-01-28T07:43:40.515729+0000", > > "last_undegraded": "2024-01-28T07:43:40.515729+0000", > > "last_fullsized": "2024-01-28T07:43:40.515729+0000", > > "mapping_epoch": 243986, > > "log_start": "80608'37589", > > "ondisk_log_start": "80608'37589", > > "created": 3726, > > "last_epoch_clean": 123203, > > "parent": "0.0", > > "parent_split_bits": 14, > > "last_scrub": "80609'38697", > > "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", > > "last_deep_scrub": "80609'38697", > > "last_deep_scrub_stamp": > "2024-01-07T08:20:59.246940+0000", > > "last_clean_scrub_stamp": > "2024-01-09T20:27:40.795531+0000", > > "objects_scrubbed": 17724, > > "log_size": 1326, > > "log_dups_size": 3000, > > "ondisk_log_size": 1326, > > "stats_invalid": false, > > "dirty_stats_invalid": false, > > "omap_stats_invalid": false, > > "hitset_stats_invalid": false, > > "hitset_bytes_stats_invalid": false, > > "pin_stats_invalid": false, > > "manifest_stats_invalid": false, > > "snaptrimq_len": 0, > > "last_scrub_duration": 29, > > "scrub_schedule": "deep scrubbing for 5005s", > > "scrub_duration": 28.446739284, > > "objects_trimmed": 0, > > "snaptrim_duration": 0, > > "stat_sum": { > > "num_bytes": 67167022618, > > "num_objects": 17743, > > "num_object_clones": 0, > > "num_object_copies": 53229, > > "num_objects_missing_on_primary": 0, > > "num_objects_missing": 0, > > "num_objects_degraded": 0, > > "num_objects_misplaced": 0, > > "num_objects_unfound": 0, > > "num_objects_dirty": 17743, > > "num_whiteouts": 0, > > "num_read": 16640, > > "num_read_kb": 30923655, > > "num_write": 55094, > > "num_write_kb": 71824558, > > "num_scrub_errors": 0, > > "num_shallow_scrub_errors": 0, > > "num_deep_scrub_errors": 0, > > "num_objects_recovered": 35403, > > "num_bytes_recovered": 134021819371, > > "num_keys_recovered": 0, > > "num_objects_omap": 0, > > "num_objects_hit_set_archive": 0, > > "num_bytes_hit_set_archive": 0, > > "num_flush": 0, > > "num_flush_kb": 0, > > "num_evict": 0, > > "num_evict_kb": 0, > > "num_promote": 0, > > "num_flush_mode_high": 0, > > "num_flush_mode_low": 0, > > "num_evict_mode_some": 0, > > "num_evict_mode_full": 0, > > "num_objects_pinned": 0, > > "num_legacy_snapsets": 0, > > "num_large_omap_objects": 0, > > "num_objects_manifest": 0, > > "num_omap_bytes": 0, > > "num_omap_keys": 0, > > "num_objects_repaired": 0 > > }, > > "up": [ > > 313, > > 643, > > 727 > > ], > > "acting": [ > > 313, > > 643, > > 717 > > ], > > "avail_no_missing": [], > > "object_location_counts": [], > > "blocked_by": [], > > "up_primary": 313, > > "acting_primary": 313, > > "purged_snaps": [] > > }, > > "empty": 0, > > "dne": 0, > > "incomplete": 0, > > "last_epoch_started": 243987, > > "hit_set_history": { > > "current_last_update": "0'0", > > "history": [] > > } > > }, > > { > > "peer": "717", > > "pgid": "6.38f1", > > "last_update": "212333'38916", > > "last_complete": "212333'38916", > > "log_tail": "80608'37589", > > "last_user_version": 0, > > "last_backfill": "MAX", > > "purged_snaps": [], > > "history": { > > "epoch_created": 3726, > > "epoch_pool_created": 3279, > > "last_epoch_started": 243987, > > "last_interval_started": 243986, > > "last_epoch_clean": 220174, > > "last_interval_clean": 220173, > > "last_epoch_split": 3726, > > "last_epoch_marked_full": 0, > > "same_up_since": 238347, > > "same_interval_since": 243986, > > "same_primary_since": 3728, > > "last_scrub": "212333'38916", > > "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "last_deep_scrub": "212333'38916", > > "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > > "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > > "prior_readable_until_ub": 0 > > }, > > "stats": { > > "version": "212331'38915", > > "reported_seq": 362849, > > "reported_epoch": 212333, > > "state": "active+clean+scrubbing+deep", > > "last_fresh": "2024-01-28T07:43:40.515729+0000", > > "last_change": "2024-01-28T06:20:14.969313+0000", > > "last_active": "2024-01-28T07:43:40.515729+0000", > > "last_peered": "2024-01-28T07:43:40.515729+0000", > > "last_clean": "2024-01-28T07:43:40.515729+0000", > > "last_became_active": "2024-01-23T10:01:07.710478+0000", > > "last_became_peered": "2024-01-23T10:01:07.710478+0000", > > "last_unstale": "2024-01-28T07:43:40.515729+0000", > > "last_undegraded": "2024-01-28T07:43:40.515729+0000", > > "last_fullsized": "2024-01-28T07:43:40.515729+0000", > > "mapping_epoch": 243986, > > "log_start": "80608'37589", > > "ondisk_log_start": "80608'37589", > > "created": 3726, > > "last_epoch_clean": 123203, > > "parent": "0.0", > > "parent_split_bits": 14, > > "last_scrub": "80609'38697", > > "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", > > "last_deep_scrub": "80609'38697", > > "last_deep_scrub_stamp": > "2024-01-07T08:20:59.246940+0000", > > "last_clean_scrub_stamp": > "2024-01-09T20:27:40.795531+0000", > > "objects_scrubbed": 17724, > > "log_size": 1326, > > "log_dups_size": 3000, > > "ondisk_log_size": 1326, > > "stats_invalid": false, > > "dirty_stats_invalid": false, > > "omap_stats_invalid": false, > > "hitset_stats_invalid": false, > > "hitset_bytes_stats_invalid": false, > > "pin_stats_invalid": false, > > "manifest_stats_invalid": false, > > "snaptrimq_len": 0, > > "last_scrub_duration": 29, > > "scrub_schedule": "deep scrubbing for 5005s", > > "scrub_duration": 28.446739284, > > "objects_trimmed": 0, > > "snaptrim_duration": 0, > > "stat_sum": { > > "num_bytes": 67167022618, > > "num_objects": 17743, > > "num_object_clones": 0, > > "num_object_copies": 53229, > > "num_objects_missing_on_primary": 0, > > "num_objects_missing": 0, > > "num_objects_degraded": 0, > > "num_objects_misplaced": 0, > > "num_objects_unfound": 0, > > "num_objects_dirty": 17743, > > "num_whiteouts": 0, > > "num_read": 16640, > > "num_read_kb": 30923655, > > "num_write": 55094, > > "num_write_kb": 71824558, > > "num_scrub_errors": 0, > > "num_shallow_scrub_errors": 0, > > "num_deep_scrub_errors": 0, > > "num_objects_recovered": 35403, > > "num_bytes_recovered": 134021819371, > > "num_keys_recovered": 0, > > "num_objects_omap": 0, > > "num_objects_hit_set_archive": 0, > > "num_bytes_hit_set_archive": 0, > > "num_flush": 0, > > "num_flush_kb": 0, > > "num_evict": 0, > > "num_evict_kb": 0, > > "num_promote": 0, > > "num_flush_mode_high": 0, > > "num_flush_mode_low": 0, > > "num_evict_mode_some": 0, > > "num_evict_mode_full": 0, > > "num_objects_pinned": 0, > > "num_legacy_snapsets": 0, > > "num_large_omap_objects": 0, > > "num_objects_manifest": 0, > > "num_omap_bytes": 0, > > "num_omap_keys": 0, > > "num_objects_repaired": 0 > > }, > > "up": [ > > 313, > > 643, > > 727 > > ], > > "acting": [ > > 313, > > 643, > > 717 > > ], > > "avail_no_missing": [], > > "object_location_counts": [], > > "blocked_by": [], > > "up_primary": 313, > > "acting_primary": 313, > > "purged_snaps": [] > > }, > > "empty": 0, > > "dne": 0, > > "incomplete": 0, > > "last_epoch_started": 243987, > > "hit_set_history": { > > "current_last_update": "0'0", > > "history": [] > > } > > }, > > { > > "peer": "726", > > "pgid": "6.38f1", > > "last_update": "212333'38916", > > "last_complete": "212333'38916", > > "log_tail": "80608'37589", > > "last_user_version": 0, > > "last_backfill": "MIN", > > "purged_snaps": [], > > "history": { > > "epoch_created": 3726, > > "epoch_pool_created": 3279, > > "last_epoch_started": 238348, > > "last_interval_started": 238347, > > "last_epoch_clean": 220174, > > "last_interval_clean": 220173, > > "last_epoch_split": 3726, > > "last_epoch_marked_full": 0, > > "same_up_since": 238347, > > "same_interval_since": 243986, > > "same_primary_since": 3728, > > "last_scrub": "212333'38916", > > "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "last_deep_scrub": "212333'38916", > > "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > > "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > > "prior_readable_until_ub": 11.834917336 > > }, > > "stats": { > > "version": "0'0", > > "reported_seq": 0, > > "reported_epoch": 0, > > "state": "unknown", > > "last_fresh": "0.000000", > > "last_change": "0.000000", > > "last_active": "0.000000", > > "last_peered": "0.000000", > > "last_clean": "0.000000", > > "last_became_active": "0.000000", > > "last_became_peered": "0.000000", > > "last_unstale": "0.000000", > > "last_undegraded": "0.000000", > > "last_fullsized": "0.000000", > > "mapping_epoch": 243986, > > "log_start": "0'0", > > "ondisk_log_start": "0'0", > > "created": 0, > > "last_epoch_clean": 0, > > "parent": "0.0", > > "parent_split_bits": 0, > > "last_scrub": "0'0", > > "last_scrub_stamp": "0.000000", > > "last_deep_scrub": "0'0", > > "last_deep_scrub_stamp": "0.000000", > > "last_clean_scrub_stamp": "0.000000", > > "objects_scrubbed": 0, > > "log_size": 0, > > "log_dups_size": 0, > > "ondisk_log_size": 0, > > "stats_invalid": false, > > "dirty_stats_invalid": false, > > "omap_stats_invalid": false, > > "hitset_stats_invalid": false, > > "hitset_bytes_stats_invalid": false, > > "pin_stats_invalid": false, > > "manifest_stats_invalid": false, > > "snaptrimq_len": 0, > > "last_scrub_duration": 0, > > "scrub_schedule": "--", > > "scrub_duration": 0, > > "objects_trimmed": 0, > > "snaptrim_duration": 0, > > "stat_sum": { > > "num_bytes": 4194304, > > "num_objects": 0, > > "num_object_clones": 0, > > "num_object_copies": 0, > > "num_objects_missing_on_primary": 0, > > "num_objects_missing": 17743, > > "num_objects_degraded": 0, > > "num_objects_misplaced": 0, > > "num_objects_unfound": 0, > > "num_objects_dirty": 0, > > "num_whiteouts": 0, > > "num_read": 0, > > "num_read_kb": 0, > > "num_write": 0, > > "num_write_kb": 0, > > "num_scrub_errors": 0, > > "num_shallow_scrub_errors": 0, > > "num_deep_scrub_errors": 0, > > "num_objects_recovered": 0, > > "num_bytes_recovered": 0, > > "num_keys_recovered": 0, > > "num_objects_omap": 0, > > "num_objects_hit_set_archive": 0, > > "num_bytes_hit_set_archive": 0, > > "num_flush": 0, > > "num_flush_kb": 0, > > "num_evict": 0, > > "num_evict_kb": 0, > > "num_promote": 0, > > "num_flush_mode_high": 0, > > "num_flush_mode_low": 0, > > "num_evict_mode_some": 0, > > "num_evict_mode_full": 0, > > "num_objects_pinned": 0, > > "num_legacy_snapsets": 0, > > "num_large_omap_objects": 0, > > "num_objects_manifest": 0, > > "num_omap_bytes": 0, > > "num_omap_keys": 0, > > "num_objects_repaired": 0 > > }, > > "up": [ > > 313, > > 643, > > 727 > > ], > > "acting": [ > > 313, > > 643, > > 717 > > ], > > "avail_no_missing": [], > > "object_location_counts": [], > > "blocked_by": [], > > "up_primary": 313, > > "acting_primary": 313, > > "purged_snaps": [] > > }, > > "empty": 0, > > "dne": 0, > > "incomplete": 1, > > "last_epoch_started": 238320, > > "hit_set_history": { > > "current_last_update": "0'0", > > "history": [] > > } > > }, > > { > > "peer": "727", > > "pgid": "6.38f1", > > "last_update": "212333'38916", > > "last_complete": "212333'38916", > > "log_tail": "80608'37589", > > "last_user_version": 0, > > "last_backfill": > "6:8f1d3293:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.V-JlvDFjbQNgn8GtBLN2LePecv5fLKf_3:head", > > "purged_snaps": [], > > "history": { > > "epoch_created": 3726, > > "epoch_pool_created": 3279, > > "last_epoch_started": 243987, > > "last_interval_started": 243986, > > "last_epoch_clean": 220174, > > "last_interval_clean": 220173, > > "last_epoch_split": 3726, > > "last_epoch_marked_full": 0, > > "same_up_since": 238347, > > "same_interval_since": 243986, > > "same_primary_since": 3728, > > "last_scrub": "212333'38916", > > "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > > "last_deep_scrub": "212333'38916", > > "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > > "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > > "prior_readable_until_ub": 0 > > }, > > "stats": { > > "version": "0'0", > > "reported_seq": 0, > > "reported_epoch": 0, > > "state": "unknown", > > "last_fresh": "0.000000", > > "last_change": "0.000000", > > "last_active": "0.000000", > > "last_peered": "0.000000", > > "last_clean": "0.000000", > > "last_became_active": "0.000000", > > "last_became_peered": "0.000000", > > "last_unstale": "0.000000", > > "last_undegraded": "0.000000", > > "last_fullsized": "0.000000", > > "mapping_epoch": 243986, > > "log_start": "0'0", > > "ondisk_log_start": "0'0", > > "created": 0, > > "last_epoch_clean": 0, > > "parent": "0.0", > > "parent_split_bits": 0, > > "last_scrub": "0'0", > > "last_scrub_stamp": "0.000000", > > "last_deep_scrub": "0'0", > > "last_deep_scrub_stamp": "0.000000", > > "last_clean_scrub_stamp": "0.000000", > > "objects_scrubbed": 0, > > "log_size": 0, > > "log_dups_size": 0, > > "ondisk_log_size": 0, > > "stats_invalid": false, > > "dirty_stats_invalid": false, > > "omap_stats_invalid": false, > > "hitset_stats_invalid": false, > > "hitset_bytes_stats_invalid": false, > > "pin_stats_invalid": false, > > "manifest_stats_invalid": false, > > "snaptrimq_len": 0, > > "last_scrub_duration": 0, > > "scrub_schedule": "--", > > "scrub_duration": 0, > > "objects_trimmed": 0, > > "snaptrim_duration": 0, > > "stat_sum": { > > "num_bytes": 20227608233, > > "num_objects": 5342, > > "num_object_clones": 0, > > "num_object_copies": 0, > > "num_objects_missing_on_primary": 0, > > "num_objects_missing": 12401, > > "num_objects_degraded": 0, > > "num_objects_misplaced": 0, > > "num_objects_unfound": 0, > > "num_objects_dirty": 5342, > > "num_whiteouts": 0, > > "num_read": 0, > > "num_read_kb": 0, > > "num_write": 0, > > "num_write_kb": 0, > > "num_scrub_errors": 0, > > "num_shallow_scrub_errors": 0, > > "num_deep_scrub_errors": 0, > > "num_objects_recovered": 0, > > "num_bytes_recovered": 0, > > "num_keys_recovered": 0, > > "num_objects_omap": 0, > > "num_objects_hit_set_archive": 0, > > "num_bytes_hit_set_archive": 0, > > "num_flush": 0, > > "num_flush_kb": 0, > > "num_evict": 0, > > "num_evict_kb": 0, > > "num_promote": 0, > > "num_flush_mode_high": 0, > > "num_flush_mode_low": 0, > > "num_evict_mode_some": 0, > > "num_evict_mode_full": 0, > > "num_objects_pinned": 0, > > "num_legacy_snapsets": 0, > > "num_large_omap_objects": 0, > > "num_objects_manifest": 0, > > "num_omap_bytes": 0, > > "num_omap_keys": 0, > > "num_objects_repaired": 0 > > }, > > "up": [ > > 313, > > 643, > > 727 > > ], > > "acting": [ > > 313, > > 643, > > 717 > > ], > > "avail_no_missing": [], > > "object_location_counts": [], > > "blocked_by": [], > > "up_primary": 313, > > "acting_primary": 313, > > "purged_snaps": [] > > }, > > "empty": 0, > > "dne": 0, > > "incomplete": 1, > > "last_epoch_started": 243987, > > "hit_set_history": { > > "current_last_update": "0'0", > > "history": [] > > } > > } > > ], > > "recovery_state": [ > > { > > "name": "Started/Primary/Active", > > "enter_time": "2024-02-03T22:31:15.023323+0000", > > "might_have_unfound": [], > > "recovery_progress": { > > "backfill_targets": [ > > "727" > > ], > > "waiting_on_backfill": [], > > "last_backfill_started": > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", > > "backfill_info": { > > "begin": > "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", > > "end": > "6:8f1d3549:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.PTxEYjlIXk2vkWpRgKUOu4ygweU-Jqy_5:head", > > "objects": [ > > { > > "object": > "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", > > "version": "80588'26367" > > }, > > { > > "object": > "6:8f1d32b0:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_18-71435:head", > > "version": "80588'26368" > > }, > > { > > "object": > "6:8f1d32c2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.o_s0NGR7mrH-4JKpvYS6Ip14hbm4w5o_5:head", > > "version": "80588'26369" > > }, > > { > > "object": > "6:8f1d32c3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WV1Ja5V21BKfmI5e1W90a7iQXdcTnSG_6:head", > > "version": "80588'26370" > > }, > > { > > "object": > "6:8f1d32c4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.96tkpnCzKmtnPTLewpai2cXSr_z_QcR_1:head", > > "version": "80588'26371" > > }, > > { > > "object": > "6:8f1d331d:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.InACTSlwV7irTe0g9cLnadT8EbQr4Cl_5:head", > > "version": "80588'26372" > > }, > > { > > "object": > "6:8f1d3326:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_233-155987:head", > > "version": "80588'26373" > > }, > > { > > "object": > "6:8f1d3332:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.5aipEOj0vV3h4jdSf6tkAUn64yJmvoH_1:head", > > "version": "80588'26374" > > }, > > { > > "object": > "6:8f1d3343:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QTzZJhDV6JLC5wJ5BDxHvK4sfp3Hr6o_6:head", > > "version": "80588'26375" > > }, > > { > > "object": > "6:8f1d334b:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QgMCmHWV87HlAjqkfvbO9X5OUZrgIPc_2:head", > > "version": "80588'26376" > > }, > > { > > "object": > "6:8f1d3355:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.H4j6DWgQa0jsrl-xzgugsW22GHCOgGL_7:head", > > "version": "80588'26377" > > }, > > { > > "object": > "6:8f1d3379:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.25g7c0w-VnGqX-z_m53YmnoWUO0R56N_5:head", > > "version": "80588'26378" > > }, > > { > > "object": > "6:8f1d337c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WMmV4BoLMm4q6BztoZ1uPc4boiJ-zhW_1:head", > > "version": "80588'26379" > > }, > > { > > "object": > "6:8f1d3385:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_184-108817:head", > > "version": "80588'26380" > > }, > > { > > "object": > "6:8f1d3389:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sGNiGV9K-TGyTYRqWSOcSIXkBBOT93X_3:head", > > "version": "80588'26381" > > }, > > { > > "object": > "6:8f1d339c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.NkMm_dhUxRyxXhBDYqGCuOaSvwNvSuJ_3:head", > > "version": "80588'26382" > > }, > > { > > "object": > "6:8f1d33a3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.BV__z8jIWqcjMsTf40MvaARU1dVtCv2_4:head", > > "version": "80588'26383" > > }, > > { > > "object": > "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.XJLSyRSnrDupnOjWzHKc4wykPJjqR5g_1:head", > > "version": "80588'26384" > > }, > > { > > "object": > "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.oYSnMakOIpFmfLdym2oCx-fLBDCLDmr_4:head", > > "version": "80588'26385" > > }, > > { > > "object": > "6:8f1d33b4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_135-2779:head", > > "version": "80588'26386" > > }, > > { > > "object": > "6:8f1d33b6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sEHqthyEGg3FQP4-vdL9zkhSJmQEaIc_2:head", > > "version": "80588'26387" > > }, > > { > > "object": > "6:8f1d33ec:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.wL36lNEfHGE6B4V-YT6fMuAI-n2uV30_1:head", > > "version": "80588'26388" > > }, > > { > > "object": > "6:8f1d33f3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.j1UloV7DpRwG0-4AKULi3CXOTC0CrdU_4:head", > > "version": "80588'26389" > > }, > > { > > "object": > "6:8f1d33f5:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.q3ExUB7Mp-LggBIsRBBKyIob574rNpG_3:head", > > "version": "80588'26390" > > }, > > { > > "object": > "6:8f1d33f9:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_72-13403:head", > > "version": "80588'26391" > > }, > > { > > "object": > "6:8f1d33fd:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_157-172390:head", > > "version": "80588'26392" > > }, > > { > > "object": > "6:8f1d3403:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Wn4ed97PvQp_oDnk_Y4xlZMke0pQ4z1_3:head", > > "version": "80588'26393" > > }, > > { > > "object": > "6:8f1d3405:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.uFg6eFSD3BCEqzJLCZlw546nDqqg80U_4:head", > > "version": "80588'26394" > > }, > > { > > "object": > "6:8f1d3436:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xPwN1sJsNFAg2Nr_Vnu4ItFgHFyu_Nx_7:head", > > "version": "80588'26395" > > }, > > { > > "object": > "6:8f1d3438:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Qgd9EXWAraBWnYcZ3OhFvepsT8r0Ksi_1:head", > > "version": "80588'26396" > > }, > > { > > "object": > "6:8f1d3464:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.1KnQBzx5u-CkXROQVUzUPfJiuTq52zH_3:head", > > "version": "80588'26397" > > }, > > { > > "object": > "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Aw2QRjM0ZXkf20MtQlhyq31hwVMEWwe_1:head", > > "version": "80588'26398" > > }, > > { > > "object": > "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xprsKnjtYu1ITOsgDtkPup9Nk6xzGTN_6:head", > > "version": "80588'26399" > > }, > > { > > "object": > "6:8f1d3475:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.jo4-jEQV1kB9kHewUYdbU1UUNX60EUm_1:head", > > "version": "80588'26400" > > }, > > { > > "object": > "6:8f1d3488:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.f9EE8lnu2FzW1_vGo9J41KY-iFpI9-f_1:head", > > "version": "80588'26401" > > }, > > { > > "object": > "6:8f1d3491:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_11-138415:head", > > "version": "80588'26402" > > }, > > { > > "object": > "6:8f1d34ac:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_33-123891:head", > > "version": "80588'26403" > > }, > > { > > "object": > "6:8f1d34ad:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.x3tIcVCAeRnOJmmHhJ2hxCjsUZ3GBsK_3:head", > > "version": "80588'26404" > > }, > > { > > "object": > "6:8f1d34bc:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.9yQELiJhCNs5Z-foJHH3vEfQ_1TZdBF_1:head", > > "version": "80588'26405" > > }, > > { > > "object": > "6:8f1d34d1:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Me9Xl2_j9z2_yE1laHWpHdJ2fXvwZDZ_1:head", > > "version": "80588'26406" > > }, > > { > > "object": > "6:8f1d34fa:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Y5VD6lU4UqjXEh5Yb9GVwusKpsqGzuG_4:head", > > "version": "80588'26407" > > }, > > { > > "object": > "6:8f1d3503:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_58-178651:head", > > "version": "80588'26408" > > }, > > { > > "object": > "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.FFyoywHasB0i44Y9ZtCi_YgtYhm8cP6_6:head", > > "version": "80588'26409" > > }, > > { > > "object": > "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.lHmY-Z-Q3Ewx7kHwCoLFoFU70Xtrgzu_1:head", > > "version": "80588'26410" > > }, > > { > > "object": > "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.ETPvUyLCwz5Bfmh22s_odAwEIMCC_Vn_2:head", > > "version": "80588'26411" > > }, > > { > > "object": > "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.EdYbzDvA-7HpfrM0DMfHRgalz8ZiLhJ_1:head", > > "version": "80588'26412" > > }, > > { > > "object": > "6:8f1d3529:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1186045.1_164-1500436:head", > > "version": "80588'26413" > > } > > ] > > }, > > "peer_backfill_info": [ > > "727", > > { > > "begin": "MAX", > > "end": "MAX", > > "objects": [] > > } > > ], > > "backfills_in_flight": [ > > > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" > > ], > > "recovering": [ > > > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" > > ], > > "pg_backend": { > > "pull_from_peer": [], > > "pushing": [ > > { > > "pushing": > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", > > "pushing_to": [ > > { > > "pushing_to": "727", > > "push_info": { > > "recovery_progress": { > > "first?": 0, > > "data_complete?": 1, > > "data_recovered_to": 4194304, > > "omap_complete?": 1, > > "omap_recovered_to": "" > > }, > > "recovery_info": { > > "object": > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", > > "at_version": "80588'26366", > > "size": "4194304", > > "object_info": { > > "oid": { > > "oid": > "d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605", > > "key": "", > > "snapid": -2, > > "hash": 4182554865, > > "max": 0, > > "pool": 6, > > "namespace": "" > > }, > > "version": "80588'26366", > > "prior_version": > "6896'10987", > > "last_reqid": > "osd.313.0:4380013", > > "user_version": 10987, > > "size": 4194304, > > "mtime": > "2023-12-19T07:17:04.766167+0000", > > "local_mtime": > "2023-12-19T07:17:04.799712+0000", > > "lost": 0, > > "flags": [ > > "dirty", > > "data_digest", > > "omap_digest" > > ], > > "truncate_seq": 0, > > "truncate_size": 0, > > "data_digest": > "0x9cdfa22f", > > "omap_digest": > "0xffffffff", > > "expected_object_size": 0, > > "expected_write_size": 0, > > "alloc_hint_flags": 0, > > "manifest": { > > "type": 0 > > }, > > "watchers": {} > > }, > > "snapset": { > > "seq": 0, > > "clones": [] > > }, > > "copy_subset": "[0~4194304]", > > "clone_subset": "{}", > > "object_exist": "0" > > } > > } > > } > > ] > > } > > ] > > } > > } > > }, > > { > > "name": "Started", > > "enter_time": "2024-02-03T22:31:13.882074+0000" > > } > > ], > > "scrubber": { > > "active": false, > > "must_scrub": false, > > "must_deep_scrub": false, > > "must_repair": false, > > "need_auto": false, > > "scrub_reg_stamp": "2024-01-30T15:39:19.044093+0000", > > "schedule": "queued for deep scrub" > > }, > > "agent_state": {} > >} > >_______________________________________________ > >ceph-users mailing list -- ceph-users@xxxxxxx > >To unsubscribe send an email to ceph-users-leave@xxxxxxx > > > _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |