I have a wired problem. After restarting OSDs in one host, 1 PG is stuck active+recovering+undersized+degraded+remapped. Version is mimic. I already restarted the primary OSD to no avail. There is no recovery IO. Any ideas what else I can do? The pool is 8+3 EC. Some diagnostic output below.
Update: After a long wait the degraded PG became clean. Still, some information about what is the cause and if I can manually do something is very much appreciated. This random issue makes it hard to estimate maintenance windows.
Thanks and best regards,
Frank
# ceph status
cluster:
id:
health: HEALTH_WARN
Degraded data redundancy: 1/8260897734 objects degraded (0.000%), 1 pg degraded
services:
mon: 5 daemons, quorum ceph-01,ceph-02,ceph-03,ceph-25,ceph-26
mgr: ceph-01(active), standbys: ceph-02, ceph-03, ceph-25, ceph-26
mds: con-fs2-4/4/4 up {0=ceph-08=up:active,1=ceph-11=up:active,2=ceph-17=up:active,3=ceph-13=up:active}, 8 up:standby
osd: 1051 osds: 1050 up, 1050 in; 1 remapped pgs
data:
pools: 13 pools, 17374 pgs
objects: 980.2 M objects, 1.8 PiB
usage: 2.2 PiB used, 9.3 PiB / 11 PiB avail
pgs: 1/8260897734 objects degraded (0.000%)
17348 active+clean
24 active+clean+scrubbing+deep
1 active+recovering+undersized+degraded+remapped
1 active+clean+scrubbing
io:
client: 168 MiB/s rd, 225 MiB/s wr, 2.42 kop/s rd, 1.52 kop/s wr
# ceph health detail
HEALTH_WARN Degraded data redundancy: 1/8261075275 objects degraded (0.000%), 1 pg degraded, 1 pg undersized
PG_DEGRADED Degraded data redundancy: 1/8261075275 objects degraded (0.000%), 1 pg degraded, 1 pg undersized
pg 19.14fd is stuck undersized for 184.337696, current state active+recovering+undersized+degraded+remapped, last acting [2147483647,846,672,526,921,797,439,517,337,547,626]
# ceph pg 19.14fd query
{
"state": "active+recovering+undersized+degraded+remapped",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 728727,
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"async_recovery_targets": [
"377(0)"
],
"acting_recovery_backfill": [
"337(8)",
"377(0)",
"439(6)",
"517(7)",
"526(3)",
"547(9)",
"626(10)",
"672(2)",
"797(5)",
"846(1)",
"921(4)"
],
"info": {
"pgid": "19.14fds1",
"last_update": "728727'267261",
"last_complete": "728727'267261",
"log_tail": "728722'264170",
"last_user_version": 267261,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728727'267261",
"reported_seq": "442688",
"reported_epoch": "728727",
"state": "active+recovering+undersized+degraded+remapped",
"last_fresh": "2021-12-07 13:32:21.135325",
"last_change": "2021-12-07 13:28:32.362446",
"last_active": "2021-12-07 13:32:21.135325",
"last_peered": "2021-12-07 13:32:21.135325",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:28:24.292648",
"last_became_peered": "2021-12-07 13:28:24.292648",
"last_unstale": "2021-12-07 13:32:21.135325",
"last_undegraded": "2021-12-07 13:28:23.727336",
"last_fullsized": "2021-12-07 13:28:23.724061",
"mapping_epoch": 728726,
"log_start": "728722'264170",
"ondisk_log_start": "728722'264170",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3091,
"ondisk_log_size": 3091,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45032078044,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 1,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74242,
"num_read_kb": 37032533,
"num_write": 238161,
"num_write_kb": 96114384,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 70,
"num_bytes_recovered": 100513548,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "337(8)",
"pgid": "19.14fds8",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "377(0)",
"pgid": "19.14fds0",
"last_update": "728727'267261",
"last_complete": "728718'263869",
"log_tail": "728510'256039",
"last_user_version": 263869,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728718'263869",
"reported_seq": "439252",
"reported_epoch": "728725",
"state": "unknown",
"last_fresh": "2021-12-07 13:28:21.687563",
"last_change": "2021-12-07 13:28:21.687563",
"last_active": "2021-12-07 13:28:21.604667",
"last_peered": "2021-12-07 13:27:42.216602",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:15:50.540333",
"last_became_peered": "2021-12-07 13:15:50.540333",
"last_unstale": "2021-12-07 13:28:21.687563",
"last_undegraded": "2021-12-07 13:28:21.687563",
"last_fullsized": "2021-12-07 13:28:21.687563",
"mapping_epoch": 728726,
"log_start": "728510'256039",
"ondisk_log_start": "728510'256039",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 7830,
"ondisk_log_size": 7830,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45027700856,
"num_objects": 16381,
"num_object_clones": 1222,
"num_object_copies": 180191,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16381,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 234770,
"num_write_kb": 96102928,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 68,
"num_bytes_recovered": 94476915,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "439(6)",
"pgid": "19.14fds6",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "517(7)",
"pgid": "19.14fds7",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "526(3)",
"pgid": "19.14fds3",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "547(9)",
"pgid": "19.14fds9",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "626(10)",
"pgid": "19.14fds10",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "672(2)",
"pgid": "19.14fds2",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "797(5)",
"pgid": "19.14fds5",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "921(4)",
"pgid": "19.14fds4",
"last_update": "728727'267261",
"last_complete": "728727'266234",
"log_tail": "728717'261270",
"last_user_version": 264329,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 613988,
"epoch_pool_created": 613988,
"last_epoch_started": 728727,
"last_interval_started": 728726,
"last_epoch_clean": 728370,
"last_interval_clean": 728369,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 728723,
"same_interval_since": 728726,
"same_primary_since": 728726,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040"
},
"stats": {
"version": "728722'264328",
"reported_seq": "439732",
"reported_epoch": "728722",
"state": "active+undersized+degraded",
"last_fresh": "2021-12-07 13:28:19.480920",
"last_change": "2021-12-07 13:27:45.346288",
"last_active": "2021-12-07 13:28:19.480920",
"last_peered": "2021-12-07 13:28:19.480920",
"last_clean": "2021-12-07 13:14:51.424263",
"last_became_active": "2021-12-07 13:27:45.095705",
"last_became_peered": "2021-12-07 13:27:45.095705",
"last_unstale": "2021-12-07 13:28:19.480920",
"last_undegraded": "2021-12-07 13:27:44.520560",
"last_fullsized": "2021-12-07 13:27:44.516885",
"mapping_epoch": 728726,
"log_start": "728717'261270",
"ondisk_log_start": "728717'261270",
"created": 613988,
"last_epoch_clean": 728370,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "725992'236413",
"last_scrub_stamp": "2021-12-06 06:41:37.044040",
"last_deep_scrub": "725958'224914",
"last_deep_scrub_stamp": "2021-12-02 19:25:13.982037",
"last_clean_scrub_stamp": "2021-12-06 06:41:37.044040",
"log_size": 3058,
"ondisk_log_size": 3058,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 45031811687,
"num_objects": 16382,
"num_object_clones": 1222,
"num_object_copies": 180202,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 16382,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 16382,
"num_whiteouts": 1200,
"num_read": 74238,
"num_read_kb": 37020245,
"num_write": 235230,
"num_write_kb": 96107361,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 69,
"num_bytes_recovered": 96444486,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0
},
"up": [
377,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"acting": [
2147483647,
846,
672,
526,
921,
797,
439,
517,
337,
547,
626
],
"blocked_by": [],
"up_primary": 377,
"acting_primary": 846,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 728727,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2021-12-07 13:28:23.724139",
"might_have_unfound": [
{
"osd": "337(8)",
"status": "already probed"
},
{
"osd": "377(0)",
"status": "already probed"
},
{
"osd": "439(6)",
"status": "already probed"
},
{
"osd": "517(7)",
"status": "already probed"
},
{
"osd": "526(3)",
"status": "already probed"
},
{
"osd": "547(9)",
"status": "already probed"
},
{
"osd": "626(10)",
"status": "already probed"
},
{
"osd": "672(2)",
"status": "already probed"
},
{
"osd": "797(5)",
"status": "already probed"
},
{
"osd": "921(4)",
"status": "already probed"
}
],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.max_end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2021-12-07 13:28:22.712350"
}
],
"agent_state": {}
}
=================
Frank Schilder
AIT Risø Campus
Bygning 109, rum S14
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
