Re: 1 PG stucked in "active+undersized+degraded for long time

[Eugen Block <eblock@xxxxxx>]

Hi,

have you tried restarting the primary OSD (currently 343)? It looks  
like this PG is part of an EC pool, are there enough hosts available,  
assuming your failure-domain is host? I assume that ceph isn't able to  
recreate the shard on a different OSD. You could share your osd tree  
and the crush rule as well as the erasure profile so we could get a  
better picture.

Thanks,
Eugen

Zitat von siddhit.renake@xxxxxxxxxx:

> Hello All,
>
> Ceph version: 14.2.5-382-g8881d33957  
> (8881d33957b54b101eae9c7627b351af10e87ee8) nautilus (stable)
>
> Issue:
> 1 PG stucked in "active+undersized+degraded for long time
> Degraded data redundancy: 44800/8717052637 objects degraded  
> (0.001%), 1 pg degraded, 1 pg undersized
>
> #ceph pg dump_stuck
> PG_STAT STATE                       UP                                
>                    UP_PRIMARY ACTING                                  
>              ACTING_PRIMARY
> 15.28f0  active+undersized+degraded  
> [2147483647,343,355,415,426,640,302,392,78,202,607]        343  
> [2147483647,343,355,415,426,640,302,392,78,202,607]            343
>
> PG Query:
> #ceph pg 15.28f0 query
>
> {
>     "state": "active+undersized+degraded",
>     "snap_trimq": "[]",
>     "snap_trimq_len": 0,
>     "epoch": 303362,
>     "up": [
>         2147483647,
>         343,
>         355,
>         415,
>         426,
>         640,
>         302,
>         392,
>         78,
>         202,
>         607
>     ],
>     "acting": [
>         2147483647,
>         343,
>         355,
>         415,
>         426,
>         640,
>         302,
>         392,
>         78,
>         202,
>         607
>     ],
>     "acting_recovery_backfill": [
>         "78(8)",
>         "202(9)",
>         "302(6)",
>         "343(1)",
>         "355(2)",
>         "392(7)",
>         "415(3)",
>         "426(4)",
>         "607(10)",
>         "640(5)"
>     ],
>     "info": {
>         "pgid": "15.28f0s1",
>         "last_update": "303161'598853",
>         "last_complete": "303161'598853",
>         "log_tail": "261289'595825",
>         "last_user_version": 598853,
>         "last_backfill": "MAX",
>         "last_backfill_bitwise": 1,
>         "purged_snaps": [],
>         "history": {
>             "epoch_created": 19841,
>             "epoch_pool_created": 16141,
>             "last_epoch_started": 303017,
>             "last_interval_started": 303016,
>             "last_epoch_clean": 250583,
>             "last_interval_clean": 250582,
>             "last_epoch_split": 19841,
>             "last_epoch_marked_full": 0,
>             "same_up_since": 303016,
>             "same_interval_since": 303016,
>             "same_primary_since": 256311,
>             "last_scrub": "255277'537760",
>             "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>             "last_deep_scrub": "255277'537756",
>             "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>             "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>         },
>         "stats": {
>             "version": "303161'598853",
>             "reported_seq": "3594551",
>             "reported_epoch": "303362",
>             "state": "active+undersized+degraded",
>             "last_fresh": "2023-06-20 19:03:59.135295",
>             "last_change": "2023-06-20 15:11:12.569114",
>             "last_active": "2023-06-20 19:03:59.135295",
>             "last_peered": "2023-06-20 19:03:59.135295",
>             "last_clean": "2021-04-11 15:21:44.271834",
>             "last_became_active": "2023-06-20 15:11:12.569114",
>             "last_became_peered": "2023-06-20 15:11:12.569114",
>             "last_unstale": "2023-06-20 19:03:59.135295",
>             "last_undegraded": "2023-06-20 15:11:10.430426",
>             "last_fullsized": "2023-06-20 15:11:10.430154",
>             "mapping_epoch": 303016,
>             "log_start": "261289'595825",
>             "ondisk_log_start": "261289'595825",
>             "created": 19841,
>             "last_epoch_clean": 250583,
>             "parent": "0.0",
>             "parent_split_bits": 14,
>             "last_scrub": "255277'537760",
>             "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>             "last_deep_scrub": "255277'537756",
>             "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>             "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>             "log_size": 3028,
>             "ondisk_log_size": 3028,
>             "stats_invalid": false,
>             "dirty_stats_invalid": false,
>             "omap_stats_invalid": false,
>             "hitset_stats_invalid": false,
>             "hitset_bytes_stats_invalid": false,
>             "pin_stats_invalid": false,
>             "manifest_stats_invalid": false,
>             "snaptrimq_len": 0,
>             "stat_sum": {
>                 "num_bytes": 54989065178,
>                 "num_objects": 44800,
>                 "num_object_clones": 0,
>                 "num_object_copies": 492800,
>                 "num_objects_missing_on_primary": 0,
>                 "num_objects_missing": 0,
>                 "num_objects_degraded": 44800,
>                 "num_objects_misplaced": 0,
>                 "num_objects_unfound": 0,
>                 "num_objects_dirty": 44800,
>                 "num_whiteouts": 0,
>                 "num_read": 201078,
>                 "num_read_kb": 30408632,
>                 "num_write": 219335,
>                 "num_write_kb": 59459782,
>                 "num_scrub_errors": 0,
>                 "num_shallow_scrub_errors": 0,
>                 "num_deep_scrub_errors": 0,
>                 "num_objects_recovered": 121970,
>                 "num_bytes_recovered": 73212503327,
>                 "num_keys_recovered": 0,
>                 "num_objects_omap": 0,
>                 "num_objects_hit_set_archive": 0,
>                 "num_bytes_hit_set_archive": 0,
>                 "num_flush": 0,
>                 "num_flush_kb": 0,
>                 "num_evict": 0,
>                 "num_evict_kb": 0,
>                 "num_promote": 0,
>                 "num_flush_mode_high": 0,
>                 "num_flush_mode_low": 0,
>                 "num_evict_mode_some": 0,
>                 "num_evict_mode_full": 0,
>                 "num_objects_pinned": 0,
>                 "num_legacy_snapsets": 0,
>                 "num_large_omap_objects": 0,
>                 "num_objects_manifest": 0,
>                 "num_omap_bytes": 0,
>                 "num_omap_keys": 0,
>                 "num_objects_repaired": 0
>             },
>             "up": [
>                 2147483647,
>                 343,
>                 355,
>                 415,
>                 426,
>                 640,
>                 302,
>                 392,
>                 78,
>                 202,
>                 607
>             ],
>             "acting": [
>                 2147483647,
>                 343,
>                 355,
>                 415,
>                 426,
>                 640,
>                 302,
>                 392,
>                 78,
>                 202,
>                 607
>             ],
>             "avail_no_missing": [
>                 "343(1)",
>                 "78(8)",
>                 "202(9)",
>                 "302(6)",
>                 "355(2)",
>                 "392(7)",
>                 "415(3)",
>                 "426(4)",
>                 "607(10)",
>                 "640(5)"
>             ],
>             "object_location_counts": [
>                 {
>                     "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                     "objects": 44800
>                 }
>             ],
>             "blocked_by": [],
>             "up_primary": 343,
>             "acting_primary": 343,
>             "purged_snaps": []
>         },
>         "empty": 0,
>         "dne": 0,
>         "incomplete": 0,
>         "last_epoch_started": 303017,
>         "hit_set_history": {
>             "current_last_update": "0'0",
>             "history": []
>         }
>     },
>     "peer_info": [
>         {
>             "peer": "78(8)",
>             "pgid": "15.28f0s8",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "202(9)",
>             "pgid": "15.28f0s9",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "302(6)",
>             "pgid": "15.28f0s6",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "355(2)",
>             "pgid": "15.28f0s2",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "392(7)",
>             "pgid": "15.28f0s7",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "415(3)",
>             "pgid": "15.28f0s3",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "426(4)",
>             "pgid": "15.28f0s4",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "607(10)",
>             "pgid": "15.28f0s10",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         },
>         {
>             "peer": "640(5)",
>             "pgid": "15.28f0s5",
>             "last_update": "303161'598853",
>             "last_complete": "303161'598853",
>             "log_tail": "261289'595825",
>             "last_user_version": 598852,
>             "last_backfill": "MAX",
>             "last_backfill_bitwise": 1,
>             "purged_snaps": [],
>             "history": {
>                 "epoch_created": 19841,
>                 "epoch_pool_created": 16141,
>                 "last_epoch_started": 303017,
>                 "last_interval_started": 303016,
>                 "last_epoch_clean": 250583,
>                 "last_interval_clean": 250582,
>                 "last_epoch_split": 19841,
>                 "last_epoch_marked_full": 0,
>                 "same_up_since": 303016,
>                 "same_interval_since": 303016,
>                 "same_primary_since": 256311,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439"
>             },
>             "stats": {
>                 "version": "302924'598852",
>                 "reported_seq": "3594069",
>                 "reported_epoch": "302924",
>                 "state": "active+undersized+degraded",
>                 "last_fresh": "2023-06-19 20:33:19.103885",
>                 "last_change": "2021-04-28 20:51:18.679339",
>                 "last_active": "2023-06-19 20:33:19.103885",
>                 "last_peered": "2023-06-19 20:33:19.103885",
>                 "last_clean": "2021-04-11 15:21:44.271834",
>                 "last_became_active": "2021-04-28 20:51:18.679339",
>                 "last_became_peered": "2021-04-28 20:51:18.679339",
>                 "last_unstale": "2023-06-19 20:33:19.103885",
>                 "last_undegraded": "2021-04-28 20:51:18.617123",
>                 "last_fullsized": "2021-04-28 20:51:18.616747",
>                 "mapping_epoch": 303016,
>                 "log_start": "261289'595825",
>                 "ondisk_log_start": "261289'595825",
>                 "created": 19841,
>                 "last_epoch_clean": 250583,
>                 "parent": "0.0",
>                 "parent_split_bits": 14,
>                 "last_scrub": "255277'537760",
>                 "last_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "last_deep_scrub": "255277'537756",
>                 "last_deep_scrub_stamp": "2021-04-10 01:42:16.182528",
>                 "last_clean_scrub_stamp": "2021-04-11 03:18:39.164439",
>                 "log_size": 3027,
>                 "ondisk_log_size": 3027,
>                 "stats_invalid": false,
>                 "dirty_stats_invalid": false,
>                 "omap_stats_invalid": false,
>                 "hitset_stats_invalid": false,
>                 "hitset_bytes_stats_invalid": false,
>                 "pin_stats_invalid": false,
>                 "manifest_stats_invalid": false,
>                 "snaptrimq_len": 0,
>                 "stat_sum": {
>                     "num_bytes": 54984870874,
>                     "num_objects": 44799,
>                     "num_object_clones": 0,
>                     "num_object_copies": 492789,
>                     "num_objects_missing_on_primary": 0,
>                     "num_objects_missing": 0,
>                     "num_objects_degraded": 44799,
>                     "num_objects_misplaced": 0,
>                     "num_objects_unfound": 0,
>                     "num_objects_dirty": 44799,
>                     "num_whiteouts": 0,
>                     "num_read": 201036,
>                     "num_read_kb": 30408590,
>                     "num_write": 219334,
>                     "num_write_kb": 59455686,
>                     "num_scrub_errors": 0,
>                     "num_shallow_scrub_errors": 0,
>                     "num_deep_scrub_errors": 0,
>                     "num_objects_recovered": 121970,
>                     "num_bytes_recovered": 73212503327,
>                     "num_keys_recovered": 0,
>                     "num_objects_omap": 0,
>                     "num_objects_hit_set_archive": 0,
>                     "num_bytes_hit_set_archive": 0,
>                     "num_flush": 0,
>                     "num_flush_kb": 0,
>                     "num_evict": 0,
>                     "num_evict_kb": 0,
>                     "num_promote": 0,
>                     "num_flush_mode_high": 0,
>                     "num_flush_mode_low": 0,
>                     "num_evict_mode_some": 0,
>                     "num_evict_mode_full": 0,
>                     "num_objects_pinned": 0,
>                     "num_legacy_snapsets": 0,
>                     "num_large_omap_objects": 0,
>                     "num_objects_manifest": 0,
>                     "num_omap_bytes": 0,
>                     "num_omap_keys": 0,
>                     "num_objects_repaired": 0
>                 },
>                 "up": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "acting": [
>                     2147483647,
>                     343,
>                     355,
>                     415,
>                     426,
>                     640,
>                     302,
>                     392,
>                     78,
>                     202,
>                     607
>                 ],
>                 "avail_no_missing": [
>                     "343(1)",
>                     "78(8)",
>                     "202(9)",
>                     "302(6)",
>                     "355(2)",
>                     "392(7)",
>                     "415(3)",
>                     "426(4)",
>                     "607(10)",
>                     "640(5)"
>                 ],
>                 "object_location_counts": [
>                     {
>                         "shards":  
> "78(8),202(9),302(6),343(1),355(2),392(7),415(3),426(4),607(10),640(5)",
>                         "objects": 44799
>                     }
>                 ],
>                 "blocked_by": [],
>                 "up_primary": 343,
>                 "acting_primary": 343,
>                 "purged_snaps": []
>             },
>             "empty": 0,
>             "dne": 0,
>             "incomplete": 0,
>             "last_epoch_started": 303017,
>             "hit_set_history": {
>                 "current_last_update": "0'0",
>                 "history": []
>             }
>         }
>     ],
>     "recovery_state": [
>         {
>             "name": "Started/Primary/Active",
>             "enter_time": "2023-06-20 15:11:10.430274",
>             "might_have_unfound": [],
>             "recovery_progress": {
>                 "backfill_targets": [],
>                 "waiting_on_backfill": [],
>                 "last_backfill_started": "MIN",
>                 "backfill_info": {
>                     "begin": "MIN",
>                     "end": "MIN",
>                     "objects": []
>                 },
>                 "peer_backfill_info": [],
>                 "backfills_in_flight": [],
>                 "recovering": [],
>                 "pg_backend": {
>                     "recovery_ops": [],
>                     "read_ops": []
>                 }
>             },
>             "scrub": {
>                 "scrubber.epoch_start": "0",
>                 "scrubber.active": false,
>                 "scrubber.state": "INACTIVE",
>                 "scrubber.start": "MIN",
>                 "scrubber.end": "MIN",
>                 "scrubber.max_end": "MIN",
>                 "scrubber.subset_last_update": "0'0",
>                 "scrubber.deep": false,
>                 "scrubber.waiting_on_whom": []
>             }
>         },
>         {
>             "name": "Started",
>             "enter_time": "2023-06-20 15:11:09.294876"
>         }
>     ],
>     "agent_state": {}
> }
>
> Require assistance to to make this PG active+clean
> _______________________________________________
> ceph-users mailing list -- ceph-users@xxxxxxx
> To unsubscribe send an email to ceph-users-leave@xxxxxxx


_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx