ECC 2+2 & 4+2 HDD only. On Tue, 20 Feb 2024, 00:25 Anthony D'Atri, <anthony.datri@xxxxxxxxx> wrote: > After wrangling with this myself, both with 17.2.7 and to an extent with > 17.2.5, I'd like to follow up here and ask: > > Those who have experienced this, were the affected PGs > > * Part of an EC pool? > * Part of an HDD pool? > * Both? > > > > > > You don't say anything about the Ceph version you are running. > > I had an similar issue with 17.2.7, and is seams to be an issue with > mclock, > > when I switch to wpq everything worked again. > > > > You can read more about it here > > > https://lists.ceph.io/hyperkitty/list/ceph-users@xxxxxxx/thread/IPHBE3DLW5ABCZHSNYOBUBSI3TLWVD22/#OE3QXLAJIY6NU7PNMGHP47UK2CBZJPUG > > > > - Kai Stian Olstad > > > > > > On Tue, Feb 06, 2024 at 06:35:26AM -0000, LeonGao wrote: > >> Hi community > >> > >> We have a new Ceph cluster deployment with 100 nodes. When we are > draining an OSD host from the cluster, we see a small amount of PGs that > cannot make any progress to the end. From the logs and metrics, it seems > like the recovery progress is stuck (0 recovery ops for several days). > Would like to get some ideas on this. Re-peering and OSD restart do resolve > to mitigate the issue but we want to get to the root cause of it as > draining and recovery happen frequently. > >> > >> I have put some debugging information below. Any help is appreciated, > thanks! > >> > >> ceph -s > >> pgs: 4210926/7380034104 objects misplaced (0.057%) > >> 41198 active+clean > >> 71 active+remapped+backfilling > >> 12 active+recovering > >> > >> One of the stuck PG: > >> 6.38f1 active+remapped+backfilling [313,643,727] > 313 [313,643,717] 313 > >> > >> PG query result: > >> > >> ceph pg 6.38f1 query > >> { > >> "snap_trimq": "[]", > >> "snap_trimq_len": 0, > >> "state": "active+remapped+backfilling", > >> "epoch": 246856, > >> "up": [ > >> 313, > >> 643, > >> 727 > >> ], > >> "acting": [ > >> 313, > >> 643, > >> 717 > >> ], > >> "backfill_targets": [ > >> "727" > >> ], > >> "acting_recovery_backfill": [ > >> "313", > >> "643", > >> "717", > >> "727" > >> ], > >> "info": { > >> "pgid": "6.38f1", > >> "last_update": "212333'38916", > >> "last_complete": "212333'38916", > >> "log_tail": "80608'37589", > >> "last_user_version": 38833, > >> "last_backfill": "MAX", > >> "purged_snaps": [], > >> "history": { > >> "epoch_created": 3726, > >> "epoch_pool_created": 3279, > >> "last_epoch_started": 243987, > >> "last_interval_started": 243986, > >> "last_epoch_clean": 220174, > >> "last_interval_clean": 220173, > >> "last_epoch_split": 3726, > >> "last_epoch_marked_full": 0, > >> "same_up_since": 238347, > >> "same_interval_since": 243986, > >> "same_primary_since": 3728, > >> "last_scrub": "212333'38916", > >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "last_deep_scrub": "212333'38916", > >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", > >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "prior_readable_until_ub": 0 > >> }, > >> "stats": { > >> "version": "212333'38916", > >> "reported_seq": 413425, > >> "reported_epoch": 246856, > >> "state": "active+remapped+backfilling", > >> "last_fresh": "2024-02-05T21:14:40.838785+0000", > >> "last_change": "2024-02-03T22:33:43.052272+0000", > >> "last_active": "2024-02-05T21:14:40.838785+0000", > >> "last_peered": "2024-02-05T21:14:40.838785+0000", > >> "last_clean": "2024-02-03T04:26:35.168232+0000", > >> "last_became_active": "2024-02-03T22:31:16.037823+0000", > >> "last_became_peered": "2024-02-03T22:31:16.037823+0000", > >> "last_unstale": "2024-02-05T21:14:40.838785+0000", > >> "last_undegraded": "2024-02-05T21:14:40.838785+0000", > >> "last_fullsized": "2024-02-05T21:14:40.838785+0000", > >> "mapping_epoch": 243986, > >> "log_start": "80608'37589", > >> "ondisk_log_start": "80608'37589", > >> "created": 3726, > >> "last_epoch_clean": 220174, > >> "parent": "0.0", > >> "parent_split_bits": 14, > >> "last_scrub": "212333'38916", > >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "last_deep_scrub": "212333'38916", > >> "last_deep_scrub_stamp": "2024-01-28T07:43:45.920198+0000", > >> "last_clean_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "objects_scrubbed": 17743, > >> "log_size": 1327, > >> "log_dups_size": 3000, > >> "ondisk_log_size": 1327, > >> "stats_invalid": false, > >> "dirty_stats_invalid": false, > >> "omap_stats_invalid": false, > >> "hitset_stats_invalid": false, > >> "hitset_bytes_stats_invalid": false, > >> "pin_stats_invalid": false, > >> "manifest_stats_invalid": false, > >> "snaptrimq_len": 0, > >> "last_scrub_duration": 29, > >> "scrub_schedule": "queued for deep scrub", > >> "scrub_duration": 28.699844475999999, > >> "objects_trimmed": 0, > >> "snaptrim_duration": 0, > >> "stat_sum": { > >> "num_bytes": 67167022618, > >> "num_objects": 17743, > >> "num_object_clones": 0, > >> "num_object_copies": 53229, > >> "num_objects_missing_on_primary": 0, > >> "num_objects_missing": 0, > >> "num_objects_degraded": 0, > >> "num_objects_misplaced": 12401, > >> "num_objects_unfound": 0, > >> "num_objects_dirty": 17743, > >> "num_whiteouts": 0, > >> "num_read": 16640, > >> "num_read_kb": 30923655, > >> "num_write": 55094, > >> "num_write_kb": 71824558, > >> "num_scrub_errors": 0, > >> "num_shallow_scrub_errors": 0, > >> "num_deep_scrub_errors": 0, > >> "num_objects_recovered": 40746, > >> "num_bytes_recovered": 154245233300, > >> "num_keys_recovered": 0, > >> "num_objects_omap": 0, > >> "num_objects_hit_set_archive": 0, > >> "num_bytes_hit_set_archive": 0, > >> "num_flush": 0, > >> "num_flush_kb": 0, > >> "num_evict": 0, > >> "num_evict_kb": 0, > >> "num_promote": 0, > >> "num_flush_mode_high": 0, > >> "num_flush_mode_low": 0, > >> "num_evict_mode_some": 0, > >> "num_evict_mode_full": 0, > >> "num_objects_pinned": 0, > >> "num_legacy_snapsets": 0, > >> "num_large_omap_objects": 0, > >> "num_objects_manifest": 0, > >> "num_omap_bytes": 0, > >> "num_omap_keys": 0, > >> "num_objects_repaired": 0 > >> }, > >> "up": [ > >> 313, > >> 643, > >> 727 > >> ], > >> "acting": [ > >> 313, > >> 643, > >> 717 > >> ], > >> "avail_no_missing": [ > >> "313", > >> "643", > >> "717" > >> ], > >> "object_location_counts": [ > >> { > >> "shards": "313,643,717", > >> "objects": 17743 > >> } > >> ], > >> "blocked_by": [], > >> "up_primary": 313, > >> "acting_primary": 313, > >> "purged_snaps": [] > >> }, > >> "empty": 0, > >> "dne": 0, > >> "incomplete": 0, > >> "last_epoch_started": 243987, > >> "hit_set_history": { > >> "current_last_update": "0'0", > >> "history": [] > >> } > >> }, > >> "peer_info": [ > >> { > >> "peer": "643", > >> "pgid": "6.38f1", > >> "last_update": "212333'38916", > >> "last_complete": "212333'38916", > >> "log_tail": "80608'37589", > >> "last_user_version": 38833, > >> "last_backfill": "MAX", > >> "purged_snaps": [], > >> "history": { > >> "epoch_created": 3726, > >> "epoch_pool_created": 3279, > >> "last_epoch_started": 243987, > >> "last_interval_started": 243986, > >> "last_epoch_clean": 220174, > >> "last_interval_clean": 220173, > >> "last_epoch_split": 3726, > >> "last_epoch_marked_full": 0, > >> "same_up_since": 238347, > >> "same_interval_since": 243986, > >> "same_primary_since": 3728, > >> "last_scrub": "212333'38916", > >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "last_deep_scrub": "212333'38916", > >> "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > >> "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > >> "prior_readable_until_ub": 0 > >> }, > >> "stats": { > >> "version": "212331'38915", > >> "reported_seq": 362849, > >> "reported_epoch": 212333, > >> "state": "active+clean+scrubbing+deep", > >> "last_fresh": "2024-01-28T07:43:40.515729+0000", > >> "last_change": "2024-01-28T06:20:14.969313+0000", > >> "last_active": "2024-01-28T07:43:40.515729+0000", > >> "last_peered": "2024-01-28T07:43:40.515729+0000", > >> "last_clean": "2024-01-28T07:43:40.515729+0000", > >> "last_became_active": "2024-01-23T10:01:07.710478+0000", > >> "last_became_peered": "2024-01-23T10:01:07.710478+0000", > >> "last_unstale": "2024-01-28T07:43:40.515729+0000", > >> "last_undegraded": "2024-01-28T07:43:40.515729+0000", > >> "last_fullsized": "2024-01-28T07:43:40.515729+0000", > >> "mapping_epoch": 243986, > >> "log_start": "80608'37589", > >> "ondisk_log_start": "80608'37589", > >> "created": 3726, > >> "last_epoch_clean": 123203, > >> "parent": "0.0", > >> "parent_split_bits": 14, > >> "last_scrub": "80609'38697", > >> "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", > >> "last_deep_scrub": "80609'38697", > >> "last_deep_scrub_stamp": > "2024-01-07T08:20:59.246940+0000", > >> "last_clean_scrub_stamp": > "2024-01-09T20:27:40.795531+0000", > >> "objects_scrubbed": 17724, > >> "log_size": 1326, > >> "log_dups_size": 3000, > >> "ondisk_log_size": 1326, > >> "stats_invalid": false, > >> "dirty_stats_invalid": false, > >> "omap_stats_invalid": false, > >> "hitset_stats_invalid": false, > >> "hitset_bytes_stats_invalid": false, > >> "pin_stats_invalid": false, > >> "manifest_stats_invalid": false, > >> "snaptrimq_len": 0, > >> "last_scrub_duration": 29, > >> "scrub_schedule": "deep scrubbing for 5005s", > >> "scrub_duration": 28.446739284, > >> "objects_trimmed": 0, > >> "snaptrim_duration": 0, > >> "stat_sum": { > >> "num_bytes": 67167022618, > >> "num_objects": 17743, > >> "num_object_clones": 0, > >> "num_object_copies": 53229, > >> "num_objects_missing_on_primary": 0, > >> "num_objects_missing": 0, > >> "num_objects_degraded": 0, > >> "num_objects_misplaced": 0, > >> "num_objects_unfound": 0, > >> "num_objects_dirty": 17743, > >> "num_whiteouts": 0, > >> "num_read": 16640, > >> "num_read_kb": 30923655, > >> "num_write": 55094, > >> "num_write_kb": 71824558, > >> "num_scrub_errors": 0, > >> "num_shallow_scrub_errors": 0, > >> "num_deep_scrub_errors": 0, > >> "num_objects_recovered": 35403, > >> "num_bytes_recovered": 134021819371, > >> "num_keys_recovered": 0, > >> "num_objects_omap": 0, > >> "num_objects_hit_set_archive": 0, > >> "num_bytes_hit_set_archive": 0, > >> "num_flush": 0, > >> "num_flush_kb": 0, > >> "num_evict": 0, > >> "num_evict_kb": 0, > >> "num_promote": 0, > >> "num_flush_mode_high": 0, > >> "num_flush_mode_low": 0, > >> "num_evict_mode_some": 0, > >> "num_evict_mode_full": 0, > >> "num_objects_pinned": 0, > >> "num_legacy_snapsets": 0, > >> "num_large_omap_objects": 0, > >> "num_objects_manifest": 0, > >> "num_omap_bytes": 0, > >> "num_omap_keys": 0, > >> "num_objects_repaired": 0 > >> }, > >> "up": [ > >> 313, > >> 643, > >> 727 > >> ], > >> "acting": [ > >> 313, > >> 643, > >> 717 > >> ], > >> "avail_no_missing": [], > >> "object_location_counts": [], > >> "blocked_by": [], > >> "up_primary": 313, > >> "acting_primary": 313, > >> "purged_snaps": [] > >> }, > >> "empty": 0, > >> "dne": 0, > >> "incomplete": 0, > >> "last_epoch_started": 243987, > >> "hit_set_history": { > >> "current_last_update": "0'0", > >> "history": [] > >> } > >> }, > >> { > >> "peer": "717", > >> "pgid": "6.38f1", > >> "last_update": "212333'38916", > >> "last_complete": "212333'38916", > >> "log_tail": "80608'37589", > >> "last_user_version": 0, > >> "last_backfill": "MAX", > >> "purged_snaps": [], > >> "history": { > >> "epoch_created": 3726, > >> "epoch_pool_created": 3279, > >> "last_epoch_started": 243987, > >> "last_interval_started": 243986, > >> "last_epoch_clean": 220174, > >> "last_interval_clean": 220173, > >> "last_epoch_split": 3726, > >> "last_epoch_marked_full": 0, > >> "same_up_since": 238347, > >> "same_interval_since": 243986, > >> "same_primary_since": 3728, > >> "last_scrub": "212333'38916", > >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "last_deep_scrub": "212333'38916", > >> "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > >> "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > >> "prior_readable_until_ub": 0 > >> }, > >> "stats": { > >> "version": "212331'38915", > >> "reported_seq": 362849, > >> "reported_epoch": 212333, > >> "state": "active+clean+scrubbing+deep", > >> "last_fresh": "2024-01-28T07:43:40.515729+0000", > >> "last_change": "2024-01-28T06:20:14.969313+0000", > >> "last_active": "2024-01-28T07:43:40.515729+0000", > >> "last_peered": "2024-01-28T07:43:40.515729+0000", > >> "last_clean": "2024-01-28T07:43:40.515729+0000", > >> "last_became_active": "2024-01-23T10:01:07.710478+0000", > >> "last_became_peered": "2024-01-23T10:01:07.710478+0000", > >> "last_unstale": "2024-01-28T07:43:40.515729+0000", > >> "last_undegraded": "2024-01-28T07:43:40.515729+0000", > >> "last_fullsized": "2024-01-28T07:43:40.515729+0000", > >> "mapping_epoch": 243986, > >> "log_start": "80608'37589", > >> "ondisk_log_start": "80608'37589", > >> "created": 3726, > >> "last_epoch_clean": 123203, > >> "parent": "0.0", > >> "parent_split_bits": 14, > >> "last_scrub": "80609'38697", > >> "last_scrub_stamp": "2024-01-09T20:27:40.795531+0000", > >> "last_deep_scrub": "80609'38697", > >> "last_deep_scrub_stamp": > "2024-01-07T08:20:59.246940+0000", > >> "last_clean_scrub_stamp": > "2024-01-09T20:27:40.795531+0000", > >> "objects_scrubbed": 17724, > >> "log_size": 1326, > >> "log_dups_size": 3000, > >> "ondisk_log_size": 1326, > >> "stats_invalid": false, > >> "dirty_stats_invalid": false, > >> "omap_stats_invalid": false, > >> "hitset_stats_invalid": false, > >> "hitset_bytes_stats_invalid": false, > >> "pin_stats_invalid": false, > >> "manifest_stats_invalid": false, > >> "snaptrimq_len": 0, > >> "last_scrub_duration": 29, > >> "scrub_schedule": "deep scrubbing for 5005s", > >> "scrub_duration": 28.446739284, > >> "objects_trimmed": 0, > >> "snaptrim_duration": 0, > >> "stat_sum": { > >> "num_bytes": 67167022618, > >> "num_objects": 17743, > >> "num_object_clones": 0, > >> "num_object_copies": 53229, > >> "num_objects_missing_on_primary": 0, > >> "num_objects_missing": 0, > >> "num_objects_degraded": 0, > >> "num_objects_misplaced": 0, > >> "num_objects_unfound": 0, > >> "num_objects_dirty": 17743, > >> "num_whiteouts": 0, > >> "num_read": 16640, > >> "num_read_kb": 30923655, > >> "num_write": 55094, > >> "num_write_kb": 71824558, > >> "num_scrub_errors": 0, > >> "num_shallow_scrub_errors": 0, > >> "num_deep_scrub_errors": 0, > >> "num_objects_recovered": 35403, > >> "num_bytes_recovered": 134021819371, > >> "num_keys_recovered": 0, > >> "num_objects_omap": 0, > >> "num_objects_hit_set_archive": 0, > >> "num_bytes_hit_set_archive": 0, > >> "num_flush": 0, > >> "num_flush_kb": 0, > >> "num_evict": 0, > >> "num_evict_kb": 0, > >> "num_promote": 0, > >> "num_flush_mode_high": 0, > >> "num_flush_mode_low": 0, > >> "num_evict_mode_some": 0, > >> "num_evict_mode_full": 0, > >> "num_objects_pinned": 0, > >> "num_legacy_snapsets": 0, > >> "num_large_omap_objects": 0, > >> "num_objects_manifest": 0, > >> "num_omap_bytes": 0, > >> "num_omap_keys": 0, > >> "num_objects_repaired": 0 > >> }, > >> "up": [ > >> 313, > >> 643, > >> 727 > >> ], > >> "acting": [ > >> 313, > >> 643, > >> 717 > >> ], > >> "avail_no_missing": [], > >> "object_location_counts": [], > >> "blocked_by": [], > >> "up_primary": 313, > >> "acting_primary": 313, > >> "purged_snaps": [] > >> }, > >> "empty": 0, > >> "dne": 0, > >> "incomplete": 0, > >> "last_epoch_started": 243987, > >> "hit_set_history": { > >> "current_last_update": "0'0", > >> "history": [] > >> } > >> }, > >> { > >> "peer": "726", > >> "pgid": "6.38f1", > >> "last_update": "212333'38916", > >> "last_complete": "212333'38916", > >> "log_tail": "80608'37589", > >> "last_user_version": 0, > >> "last_backfill": "MIN", > >> "purged_snaps": [], > >> "history": { > >> "epoch_created": 3726, > >> "epoch_pool_created": 3279, > >> "last_epoch_started": 238348, > >> "last_interval_started": 238347, > >> "last_epoch_clean": 220174, > >> "last_interval_clean": 220173, > >> "last_epoch_split": 3726, > >> "last_epoch_marked_full": 0, > >> "same_up_since": 238347, > >> "same_interval_since": 243986, > >> "same_primary_since": 3728, > >> "last_scrub": "212333'38916", > >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "last_deep_scrub": "212333'38916", > >> "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > >> "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > >> "prior_readable_until_ub": 11.834917336 > >> }, > >> "stats": { > >> "version": "0'0", > >> "reported_seq": 0, > >> "reported_epoch": 0, > >> "state": "unknown", > >> "last_fresh": "0.000000", > >> "last_change": "0.000000", > >> "last_active": "0.000000", > >> "last_peered": "0.000000", > >> "last_clean": "0.000000", > >> "last_became_active": "0.000000", > >> "last_became_peered": "0.000000", > >> "last_unstale": "0.000000", > >> "last_undegraded": "0.000000", > >> "last_fullsized": "0.000000", > >> "mapping_epoch": 243986, > >> "log_start": "0'0", > >> "ondisk_log_start": "0'0", > >> "created": 0, > >> "last_epoch_clean": 0, > >> "parent": "0.0", > >> "parent_split_bits": 0, > >> "last_scrub": "0'0", > >> "last_scrub_stamp": "0.000000", > >> "last_deep_scrub": "0'0", > >> "last_deep_scrub_stamp": "0.000000", > >> "last_clean_scrub_stamp": "0.000000", > >> "objects_scrubbed": 0, > >> "log_size": 0, > >> "log_dups_size": 0, > >> "ondisk_log_size": 0, > >> "stats_invalid": false, > >> "dirty_stats_invalid": false, > >> "omap_stats_invalid": false, > >> "hitset_stats_invalid": false, > >> "hitset_bytes_stats_invalid": false, > >> "pin_stats_invalid": false, > >> "manifest_stats_invalid": false, > >> "snaptrimq_len": 0, > >> "last_scrub_duration": 0, > >> "scrub_schedule": "--", > >> "scrub_duration": 0, > >> "objects_trimmed": 0, > >> "snaptrim_duration": 0, > >> "stat_sum": { > >> "num_bytes": 4194304, > >> "num_objects": 0, > >> "num_object_clones": 0, > >> "num_object_copies": 0, > >> "num_objects_missing_on_primary": 0, > >> "num_objects_missing": 17743, > >> "num_objects_degraded": 0, > >> "num_objects_misplaced": 0, > >> "num_objects_unfound": 0, > >> "num_objects_dirty": 0, > >> "num_whiteouts": 0, > >> "num_read": 0, > >> "num_read_kb": 0, > >> "num_write": 0, > >> "num_write_kb": 0, > >> "num_scrub_errors": 0, > >> "num_shallow_scrub_errors": 0, > >> "num_deep_scrub_errors": 0, > >> "num_objects_recovered": 0, > >> "num_bytes_recovered": 0, > >> "num_keys_recovered": 0, > >> "num_objects_omap": 0, > >> "num_objects_hit_set_archive": 0, > >> "num_bytes_hit_set_archive": 0, > >> "num_flush": 0, > >> "num_flush_kb": 0, > >> "num_evict": 0, > >> "num_evict_kb": 0, > >> "num_promote": 0, > >> "num_flush_mode_high": 0, > >> "num_flush_mode_low": 0, > >> "num_evict_mode_some": 0, > >> "num_evict_mode_full": 0, > >> "num_objects_pinned": 0, > >> "num_legacy_snapsets": 0, > >> "num_large_omap_objects": 0, > >> "num_objects_manifest": 0, > >> "num_omap_bytes": 0, > >> "num_omap_keys": 0, > >> "num_objects_repaired": 0 > >> }, > >> "up": [ > >> 313, > >> 643, > >> 727 > >> ], > >> "acting": [ > >> 313, > >> 643, > >> 717 > >> ], > >> "avail_no_missing": [], > >> "object_location_counts": [], > >> "blocked_by": [], > >> "up_primary": 313, > >> "acting_primary": 313, > >> "purged_snaps": [] > >> }, > >> "empty": 0, > >> "dne": 0, > >> "incomplete": 1, > >> "last_epoch_started": 238320, > >> "hit_set_history": { > >> "current_last_update": "0'0", > >> "history": [] > >> } > >> }, > >> { > >> "peer": "727", > >> "pgid": "6.38f1", > >> "last_update": "212333'38916", > >> "last_complete": "212333'38916", > >> "log_tail": "80608'37589", > >> "last_user_version": 0, > >> "last_backfill": > "6:8f1d3293:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.V-JlvDFjbQNgn8GtBLN2LePecv5fLKf_3:head", > >> "purged_snaps": [], > >> "history": { > >> "epoch_created": 3726, > >> "epoch_pool_created": 3279, > >> "last_epoch_started": 243987, > >> "last_interval_started": 243986, > >> "last_epoch_clean": 220174, > >> "last_interval_clean": 220173, > >> "last_epoch_split": 3726, > >> "last_epoch_marked_full": 0, > >> "same_up_since": 238347, > >> "same_interval_since": 243986, > >> "same_primary_since": 3728, > >> "last_scrub": "212333'38916", > >> "last_scrub_stamp": "2024-01-29T13:43:10.654709+0000", > >> "last_deep_scrub": "212333'38916", > >> "last_deep_scrub_stamp": > "2024-01-28T07:43:45.920198+0000", > >> "last_clean_scrub_stamp": > "2024-01-29T13:43:10.654709+0000", > >> "prior_readable_until_ub": 0 > >> }, > >> "stats": { > >> "version": "0'0", > >> "reported_seq": 0, > >> "reported_epoch": 0, > >> "state": "unknown", > >> "last_fresh": "0.000000", > >> "last_change": "0.000000", > >> "last_active": "0.000000", > >> "last_peered": "0.000000", > >> "last_clean": "0.000000", > >> "last_became_active": "0.000000", > >> "last_became_peered": "0.000000", > >> "last_unstale": "0.000000", > >> "last_undegraded": "0.000000", > >> "last_fullsized": "0.000000", > >> "mapping_epoch": 243986, > >> "log_start": "0'0", > >> "ondisk_log_start": "0'0", > >> "created": 0, > >> "last_epoch_clean": 0, > >> "parent": "0.0", > >> "parent_split_bits": 0, > >> "last_scrub": "0'0", > >> "last_scrub_stamp": "0.000000", > >> "last_deep_scrub": "0'0", > >> "last_deep_scrub_stamp": "0.000000", > >> "last_clean_scrub_stamp": "0.000000", > >> "objects_scrubbed": 0, > >> "log_size": 0, > >> "log_dups_size": 0, > >> "ondisk_log_size": 0, > >> "stats_invalid": false, > >> "dirty_stats_invalid": false, > >> "omap_stats_invalid": false, > >> "hitset_stats_invalid": false, > >> "hitset_bytes_stats_invalid": false, > >> "pin_stats_invalid": false, > >> "manifest_stats_invalid": false, > >> "snaptrimq_len": 0, > >> "last_scrub_duration": 0, > >> "scrub_schedule": "--", > >> "scrub_duration": 0, > >> "objects_trimmed": 0, > >> "snaptrim_duration": 0, > >> "stat_sum": { > >> "num_bytes": 20227608233, > >> "num_objects": 5342, > >> "num_object_clones": 0, > >> "num_object_copies": 0, > >> "num_objects_missing_on_primary": 0, > >> "num_objects_missing": 12401, > >> "num_objects_degraded": 0, > >> "num_objects_misplaced": 0, > >> "num_objects_unfound": 0, > >> "num_objects_dirty": 5342, > >> "num_whiteouts": 0, > >> "num_read": 0, > >> "num_read_kb": 0, > >> "num_write": 0, > >> "num_write_kb": 0, > >> "num_scrub_errors": 0, > >> "num_shallow_scrub_errors": 0, > >> "num_deep_scrub_errors": 0, > >> "num_objects_recovered": 0, > >> "num_bytes_recovered": 0, > >> "num_keys_recovered": 0, > >> "num_objects_omap": 0, > >> "num_objects_hit_set_archive": 0, > >> "num_bytes_hit_set_archive": 0, > >> "num_flush": 0, > >> "num_flush_kb": 0, > >> "num_evict": 0, > >> "num_evict_kb": 0, > >> "num_promote": 0, > >> "num_flush_mode_high": 0, > >> "num_flush_mode_low": 0, > >> "num_evict_mode_some": 0, > >> "num_evict_mode_full": 0, > >> "num_objects_pinned": 0, > >> "num_legacy_snapsets": 0, > >> "num_large_omap_objects": 0, > >> "num_objects_manifest": 0, > >> "num_omap_bytes": 0, > >> "num_omap_keys": 0, > >> "num_objects_repaired": 0 > >> }, > >> "up": [ > >> 313, > >> 643, > >> 727 > >> ], > >> "acting": [ > >> 313, > >> 643, > >> 717 > >> ], > >> "avail_no_missing": [], > >> "object_location_counts": [], > >> "blocked_by": [], > >> "up_primary": 313, > >> "acting_primary": 313, > >> "purged_snaps": [] > >> }, > >> "empty": 0, > >> "dne": 0, > >> "incomplete": 1, > >> "last_epoch_started": 243987, > >> "hit_set_history": { > >> "current_last_update": "0'0", > >> "history": [] > >> } > >> } > >> ], > >> "recovery_state": [ > >> { > >> "name": "Started/Primary/Active", > >> "enter_time": "2024-02-03T22:31:15.023323+0000", > >> "might_have_unfound": [], > >> "recovery_progress": { > >> "backfill_targets": [ > >> "727" > >> ], > >> "waiting_on_backfill": [], > >> "last_backfill_started": > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", > >> "backfill_info": { > >> "begin": > "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", > >> "end": > "6:8f1d3549:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.PTxEYjlIXk2vkWpRgKUOu4ygweU-Jqy_5:head", > >> "objects": [ > >> { > >> "object": > "6:8f1d32a2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_121-19698:head", > >> "version": "80588'26367" > >> }, > >> { > >> "object": > "6:8f1d32b0:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_18-71435:head", > >> "version": "80588'26368" > >> }, > >> { > >> "object": > "6:8f1d32c2:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.o_s0NGR7mrH-4JKpvYS6Ip14hbm4w5o_5:head", > >> "version": "80588'26369" > >> }, > >> { > >> "object": > "6:8f1d32c3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WV1Ja5V21BKfmI5e1W90a7iQXdcTnSG_6:head", > >> "version": "80588'26370" > >> }, > >> { > >> "object": > "6:8f1d32c4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.96tkpnCzKmtnPTLewpai2cXSr_z_QcR_1:head", > >> "version": "80588'26371" > >> }, > >> { > >> "object": > "6:8f1d331d:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.InACTSlwV7irTe0g9cLnadT8EbQr4Cl_5:head", > >> "version": "80588'26372" > >> }, > >> { > >> "object": > "6:8f1d3326:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_233-155987:head", > >> "version": "80588'26373" > >> }, > >> { > >> "object": > "6:8f1d3332:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.5aipEOj0vV3h4jdSf6tkAUn64yJmvoH_1:head", > >> "version": "80588'26374" > >> }, > >> { > >> "object": > "6:8f1d3343:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QTzZJhDV6JLC5wJ5BDxHvK4sfp3Hr6o_6:head", > >> "version": "80588'26375" > >> }, > >> { > >> "object": > "6:8f1d334b:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.QgMCmHWV87HlAjqkfvbO9X5OUZrgIPc_2:head", > >> "version": "80588'26376" > >> }, > >> { > >> "object": > "6:8f1d3355:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.H4j6DWgQa0jsrl-xzgugsW22GHCOgGL_7:head", > >> "version": "80588'26377" > >> }, > >> { > >> "object": > "6:8f1d3379:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.25g7c0w-VnGqX-z_m53YmnoWUO0R56N_5:head", > >> "version": "80588'26378" > >> }, > >> { > >> "object": > "6:8f1d337c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.WMmV4BoLMm4q6BztoZ1uPc4boiJ-zhW_1:head", > >> "version": "80588'26379" > >> }, > >> { > >> "object": > "6:8f1d3385:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_184-108817:head", > >> "version": "80588'26380" > >> }, > >> { > >> "object": > "6:8f1d3389:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sGNiGV9K-TGyTYRqWSOcSIXkBBOT93X_3:head", > >> "version": "80588'26381" > >> }, > >> { > >> "object": > "6:8f1d339c:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.NkMm_dhUxRyxXhBDYqGCuOaSvwNvSuJ_3:head", > >> "version": "80588'26382" > >> }, > >> { > >> "object": > "6:8f1d33a3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.BV__z8jIWqcjMsTf40MvaARU1dVtCv2_4:head", > >> "version": "80588'26383" > >> }, > >> { > >> "object": > "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.XJLSyRSnrDupnOjWzHKc4wykPJjqR5g_1:head", > >> "version": "80588'26384" > >> }, > >> { > >> "object": > "6:8f1d33a6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.oYSnMakOIpFmfLdym2oCx-fLBDCLDmr_4:head", > >> "version": "80588'26385" > >> }, > >> { > >> "object": > "6:8f1d33b4:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1_135-2779:head", > >> "version": "80588'26386" > >> }, > >> { > >> "object": > "6:8f1d33b6:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.sEHqthyEGg3FQP4-vdL9zkhSJmQEaIc_2:head", > >> "version": "80588'26387" > >> }, > >> { > >> "object": > "6:8f1d33ec:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.wL36lNEfHGE6B4V-YT6fMuAI-n2uV30_1:head", > >> "version": "80588'26388" > >> }, > >> { > >> "object": > "6:8f1d33f3:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.j1UloV7DpRwG0-4AKULi3CXOTC0CrdU_4:head", > >> "version": "80588'26389" > >> }, > >> { > >> "object": > "6:8f1d33f5:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.q3ExUB7Mp-LggBIsRBBKyIob574rNpG_3:head", > >> "version": "80588'26390" > >> }, > >> { > >> "object": > "6:8f1d33f9:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_72-13403:head", > >> "version": "80588'26391" > >> }, > >> { > >> "object": > "6:8f1d33fd:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_157-172390:head", > >> "version": "80588'26392" > >> }, > >> { > >> "object": > "6:8f1d3403:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Wn4ed97PvQp_oDnk_Y4xlZMke0pQ4z1_3:head", > >> "version": "80588'26393" > >> }, > >> { > >> "object": > "6:8f1d3405:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.uFg6eFSD3BCEqzJLCZlw546nDqqg80U_4:head", > >> "version": "80588'26394" > >> }, > >> { > >> "object": > "6:8f1d3436:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xPwN1sJsNFAg2Nr_Vnu4ItFgHFyu_Nx_7:head", > >> "version": "80588'26395" > >> }, > >> { > >> "object": > "6:8f1d3438:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Qgd9EXWAraBWnYcZ3OhFvepsT8r0Ksi_1:head", > >> "version": "80588'26396" > >> }, > >> { > >> "object": > "6:8f1d3464:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.1KnQBzx5u-CkXROQVUzUPfJiuTq52zH_3:head", > >> "version": "80588'26397" > >> }, > >> { > >> "object": > "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Aw2QRjM0ZXkf20MtQlhyq31hwVMEWwe_1:head", > >> "version": "80588'26398" > >> }, > >> { > >> "object": > "6:8f1d3471:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.xprsKnjtYu1ITOsgDtkPup9Nk6xzGTN_6:head", > >> "version": "80588'26399" > >> }, > >> { > >> "object": > "6:8f1d3475:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.jo4-jEQV1kB9kHewUYdbU1UUNX60EUm_1:head", > >> "version": "80588'26400" > >> }, > >> { > >> "object": > "6:8f1d3488:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.f9EE8lnu2FzW1_vGo9J41KY-iFpI9-f_1:head", > >> "version": "80588'26401" > >> }, > >> { > >> "object": > "6:8f1d3491:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_11-138415:head", > >> "version": "80588'26402" > >> }, > >> { > >> "object": > "6:8f1d34ac:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_33-123891:head", > >> "version": "80588'26403" > >> }, > >> { > >> "object": > "6:8f1d34ad:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.x3tIcVCAeRnOJmmHhJ2hxCjsUZ3GBsK_3:head", > >> "version": "80588'26404" > >> }, > >> { > >> "object": > "6:8f1d34bc:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1157644.1__shadow_.9yQELiJhCNs5Z-foJHH3vEfQ_1TZdBF_1:head", > >> "version": "80588'26405" > >> }, > >> { > >> "object": > "6:8f1d34d1:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Me9Xl2_j9z2_yE1laHWpHdJ2fXvwZDZ_1:head", > >> "version": "80588'26406" > >> }, > >> { > >> "object": > "6:8f1d34fa:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.Y5VD6lU4UqjXEh5Yb9GVwusKpsqGzuG_4:head", > >> "version": "80588'26407" > >> }, > >> { > >> "object": > "6:8f1d3503:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_58-178651:head", > >> "version": "80588'26408" > >> }, > >> { > >> "object": > "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.FFyoywHasB0i44Y9ZtCi_YgtYhm8cP6_6:head", > >> "version": "80588'26409" > >> }, > >> { > >> "object": > "6:8f1d3521:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.lHmY-Z-Q3Ewx7kHwCoLFoFU70Xtrgzu_1:head", > >> "version": "80588'26410" > >> }, > >> { > >> "object": > "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.ETPvUyLCwz5Bfmh22s_odAwEIMCC_Vn_2:head", > >> "version": "80588'26411" > >> }, > >> { > >> "object": > "6:8f1d3522:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1__shadow_.EdYbzDvA-7HpfrM0DMfHRgalz8ZiLhJ_1:head", > >> "version": "80588'26412" > >> }, > >> { > >> "object": > "6:8f1d3529:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.1186045.1_164-1500436:head", > >> "version": "80588'26413" > >> } > >> ] > >> }, > >> "peer_backfill_info": [ > >> "727", > >> { > >> "begin": "MAX", > >> "end": "MAX", > >> "objects": [] > >> } > >> ], > >> "backfills_in_flight": [ > >> > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" > >> ], > >> "recovering": [ > >> > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head" > >> ], > >> "pg_backend": { > >> "pull_from_peer": [], > >> "pushing": [ > >> { > >> "pushing": > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", > >> "pushing_to": [ > >> { > >> "pushing_to": "727", > >> "push_info": { > >> "recovery_progress": { > >> "first?": 0, > >> "data_complete?": 1, > >> "data_recovered_to": 4194304, > >> "omap_complete?": 1, > >> "omap_recovered_to": "" > >> }, > >> "recovery_info": { > >> "object": > "6:8f1d329f:::d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605:head", > >> "at_version": "80588'26366", > >> "size": "4194304", > >> "object_info": { > >> "oid": { > >> "oid": > "d0950b5f-c248-4a0e-8fed-6ebbd0bb2820.970076.1_85-22605", > >> "key": "", > >> "snapid": -2, > >> "hash": 4182554865, > >> "max": 0, > >> "pool": 6, > >> "namespace": "" > >> }, > >> "version": "80588'26366", > >> "prior_version": > "6896'10987", > >> "last_reqid": > "osd.313.0:4380013", > >> "user_version": 10987, > >> "size": 4194304, > >> "mtime": > "2023-12-19T07:17:04.766167+0000", > >> "local_mtime": > "2023-12-19T07:17:04.799712+0000", > >> "lost": 0, > >> "flags": [ > >> "dirty", > >> "data_digest", > >> "omap_digest" > >> ], > >> "truncate_seq": 0, > >> "truncate_size": 0, > >> "data_digest": > "0x9cdfa22f", > >> "omap_digest": > "0xffffffff", > >> "expected_object_size": 0, > >> "expected_write_size": 0, > >> "alloc_hint_flags": 0, > >> "manifest": { > >> "type": 0 > >> }, > >> "watchers": {} > >> }, > >> "snapset": { > >> "seq": 0, > >> "clones": [] > >> }, > >> "copy_subset": "[0~4194304]", > >> "clone_subset": "{}", > >> "object_exist": "0" > >> } > >> } > >> } > >> ] > >> } > >> ] > >> } > >> } > >> }, > >> { > >> "name": "Started", > >> "enter_time": "2024-02-03T22:31:13.882074+0000" > >> } > >> ], > >> "scrubber": { > >> "active": false, > >> "must_scrub": false, > >> "must_deep_scrub": false, > >> "must_repair": false, > >> "need_auto": false, > >> "scrub_reg_stamp": "2024-01-30T15:39:19.044093+0000", > >> "schedule": "queued for deep scrub" > >> }, > >> "agent_state": {} > >> } > >> _______________________________________________ > >> ceph-users mailing list -- ceph-users@xxxxxxx > >> To unsubscribe send an email to ceph-users-leave@xxxxxxx > >> > > _______________________________________________ > > ceph-users mailing list -- ceph-users@xxxxxxx > > To unsubscribe send an email to ceph-users-leave@xxxxxxx > _______________________________________________ > ceph-users mailing list -- ceph-users@xxxxxxx > To unsubscribe send an email to ceph-users-leave@xxxxxxx > _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |