I've been trying to get ceph to force the PG to a good state but it
continues to give me a single PG peering. This is a rook-ceph cluster on
VMs (hosts went out for a brief period) and I can't figure out how to get
this 1GB or so of data to become available to the client. This occurred
during a cluster expansion. This gave the added joy of clock skew. Another
3 disks were being added to extend the cluster storage. This is a 3 node
cluster with 6 200GB Disks each node.
cluster:
id: 31689324-f5ba-4aa4-8244-aa09a1119dc3
health: HEALTH_OK
services:
mon: 3 daemons, quorum a,c,d (age 41m)
mgr: a(active, since 2h)
osd: 18 osds: 18 up (since 17m), 18 in (since 2h); 1 remapped pgs
data:
pools: 2 pools, 768 pgs
objects: 184.38k objects, 707 GiB
usage: 1.4 TiB used, 2.1 TiB / 3.5 TiB avail
pgs: 0.130% pgs not active
767 active+clean
1 peering
io:
client: 14 KiB/s wr, 0 op/s rd, 1 op/s wr
progress:
Global Recovery Event (2h)
[===========================.] (remaining: 9s)
PG map for problematic PG
[root@rook-ceph-tools-5b54fb98c-kjt5t /]# ceph pg map 2.11c
osdmap e11328 pg 2.11c (2.11c) -> up [17,12] acting [4]
Don't see OSD 4 in pg dump:
[root@rook-ceph-tools-5b54fb98c-kjt5t /]# ceph pg dump 2> /dev/null | egrep
'2\.11c|MISS'
PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES
OMAP_BYTES* OMAP_KEYS* LOG DISK_LOG STATE STATE_STAMP
VERSION REPORTED UP UP_PRIMARY
ACTING ACTING_PRIMARY LAST_SCRUB SCN
2.11c 249 0 0 0 0
1023758336 0 0 5352 5352 peering
2021-12-08T22:44:36.691204+0000 3669'8643974 11329:7414
[17,12] 17 [17,12] 17 3162'8635517 200
ceph pg 2.11c query
{
"snap_trimq": "[]",
"snap_trimq_len": 0,
"state": "unknown",
"epoch": 12191,
"up": [
17,
12
],
"acting": [
17,
12
],
"info": {
"pgid": "2.11c",
"last_update": "3669'8643974",
"last_complete": "3669'8643974",
"log_tail": "3174'8638622",
"last_user_version": 8643974,
"last_backfill":
"2:38e2dedc:::rbd_data.1d091ec6540582.000000000000158e:head",
"purged_snaps": [],
"history": {
"epoch_created": 3286,
"epoch_pool_created": 13,
"last_epoch_started": 3725,
"last_interval_started": 3724,
"last_epoch_clean": 3544,
"last_interval_clean": 3543,
"last_epoch_split": 3286,
"last_epoch_marked_full": 0,
"same_up_since": 9923,
"same_interval_since": 12191,
"same_primary_since": 12191,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "3669'8643974",
"reported_seq": 8277,
"reported_epoch": 12191,
"state": "peering",
"last_fresh": "2021-12-08T22:59:01.450813+0000",
"last_change": "2021-12-08T22:59:01.450813+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2021-12-08T22:59:01.450813+0000",
"last_undegraded": "2021-12-08T22:59:01.450813+0000",
"last_fullsized": "2021-12-08T22:59:01.450813+0000",
"mapping_epoch": 12191,
"log_start": "3174'8638622",
"ondisk_log_start": "3174'8638622",
"created": 3286,
"last_epoch_clean": 3544,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"log_size": 5352,
"ondisk_log_size": 5352,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1023758336,
"num_objects": 249,
"num_object_clones": 0,
"num_object_copies": 498,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 249,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
17,
12
],
"acting": [
17,
12
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
0,
1,
4,
6,
7,
12
],
"up_primary": 17,
"acting_primary": 17,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 3565,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "0",
"pgid": "2.11c",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 3286,
"epoch_pool_created": 13,
"last_epoch_started": 3725,
"last_interval_started": 3724,
"last_epoch_clean": 3544,
"last_interval_clean": 3543,
"last_epoch_split": 3286,
"last_epoch_marked_full": 0,
"same_up_since": 9923,
"same_interval_since": 12191,
"same_primary_since": 12191,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 2,
"reported_epoch": 5384,
"state": "peering",
"last_fresh": "2021-12-08T20:54:53.732167+0000",
"last_change": "2021-12-08T20:54:53.732009+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2021-12-08T20:54:53.732167+0000",
"last_undegraded": "2021-12-08T20:54:53.732167+0000",
"last_fullsized": "2021-12-08T20:54:53.732167+0000",
"mapping_epoch": 12191,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 3286,
"last_epoch_clean": 3544,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
17,
12
],
"acting": [
17,
12
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
1,
6,
7,
12
],
"up_primary": 17,
"acting_primary": 17,
"purged_snaps": []
},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "1",
"pgid": "2.11c",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 0,
"epoch_pool_created": 0,
"last_epoch_started": 0,
"last_interval_started": 0,
"last_epoch_clean": 0,
"last_interval_clean": 0,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [],
"acting": [],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1,
"purged_snaps": []
},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "4",
"pgid": "2.11c",
"last_update": "3425'8643930",
"last_complete": "3425'8643930",
"log_tail": "3174'8638622",
"last_user_version": 8643930,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 3286,
"epoch_pool_created": 13,
"last_epoch_started": 3725,
"last_interval_started": 3724,
"last_epoch_clean": 3544,
"last_interval_clean": 3543,
"last_epoch_split": 3286,
"last_epoch_marked_full": 0,
"same_up_since": 9923,
"same_interval_since": 12191,
"same_primary_since": 12191,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "3425'8643930",
"reported_seq": 11928359,
"reported_epoch": 12190,
"state": "remapped+peering",
"last_fresh": "2021-12-08T22:59:00.436034+0000",
"last_change": "2021-12-08T22:59:00.436034+0000",
"last_active": "2021-12-07T20:45:51.712823+0000",
"last_peered": "2021-12-07T20:22:45.680619+0000",
"last_clean": "2021-12-07T20:17:36.452233+0000",
"last_became_active": "2021-12-07T20:17:39.677192+0000",
"last_became_peered": "2021-12-07T20:17:39.677192+0000",
"last_unstale": "2021-12-08T22:59:00.436034+0000",
"last_undegraded": "2021-12-08T22:59:00.436034+0000",
"last_fullsized": "2021-12-08T22:59:00.436034+0000",
"mapping_epoch": 12191,
"log_start": "3174'8638622",
"ondisk_log_start": "3174'8638622",
"created": 3286,
"last_epoch_clean": 3544,
"parent": "0.0",
"parent_split_bits": 9,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"log_size": 5308,
"ondisk_log_size": 5308,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1466349568,
"num_objects": 358,
"num_object_clones": 0,
"num_object_copies": 716,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 358,
"num_whiteouts": 0,
"num_read": 442291,
"num_read_kb": 6946464,
"num_write": 1084007,
"num_write_kb": 19438544,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 711,
"num_bytes_recovered": 2800112640,
"num_keys_recovered": 5,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 132,
"num_omap_keys": 4,
"num_objects_repaired": 0
},
"up": [
17,
12
],
"acting": [
17,
12
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
0,
1,
6,
7,
12,
17
],
"up_primary": 17,
"acting_primary": 17,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 3384,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "6",
"pgid": "2.11c",
"last_update": "3669'8643974",
"last_complete": "3669'8643974",
"log_tail": "3174'8638622",
"last_user_version": 0,
"last_backfill": "MIN",
"purged_snaps": [],
"history": {
"epoch_created": 3286,
"epoch_pool_created": 13,
"last_epoch_started": 3725,
"last_interval_started": 3724,
"last_epoch_clean": 3544,
"last_interval_clean": 3543,
"last_epoch_split": 3286,
"last_epoch_marked_full": 0,
"same_up_since": 9923,
"same_interval_since": 12191,
"same_primary_since": 12191,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 12191,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
17,
12
],
"acting": [
17,
12
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": 17,
"acting_primary": 17,
"purged_snaps": []
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 3725,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "7",
"pgid": "2.11c",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 0,
"epoch_pool_created": 0,
"last_epoch_started": 0,
"last_interval_started": 0,
"last_epoch_clean": 0,
"last_interval_clean": 0,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 0,
"reported_epoch": 0,
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [],
"acting": [],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1,
"purged_snaps": []
},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "12",
"pgid": "2.11c",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": [],
"history": {
"epoch_created": 3286,
"epoch_pool_created": 13,
"last_epoch_started": 3725,
"last_interval_started": 3724,
"last_epoch_clean": 3544,
"last_interval_clean": 3543,
"last_epoch_split": 3286,
"last_epoch_marked_full": 0,
"same_up_since": 9923,
"same_interval_since": 12191,
"same_primary_since": 12191,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"prior_readable_until_ub": 0
},
"stats": {
"version": "0'0",
"reported_seq": 386,
"reported_epoch": 9872,
"state": "down",
"last_fresh": "2021-12-08T22:10:55.852651+0000",
"last_change": "2021-12-08T22:10:54.848737+0000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2021-12-08T22:10:55.852651+0000",
"last_undegraded": "2021-12-08T22:10:55.852651+0000",
"last_fullsized": "2021-12-08T22:10:55.852651+0000",
"mapping_epoch": 12191,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 3286,
"last_epoch_clean": 3544,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "3162'8635517",
"last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"last_deep_scrub": "3122'8608768",
"last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
"last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"manifest_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0,
"num_objects_manifest": 0,
"num_omap_bytes": 0,
"num_omap_keys": 0,
"num_objects_repaired": 0
},
"up": [
17,
12
],
"acting": [
17,
12
],
"avail_no_missing": [],
"object_location_counts": [],
"blocked_by": [
17
],
"up_primary": 17,
"acting_primary": 17,
"purged_snaps": []
},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/WaitActingChange",
"enter_time": "2021-12-08T22:59:01.462157+0000",
"comment": "waiting for pg acting set to change"
},
{
"name": "Started",
"enter_time": "2021-12-08T22:59:01.450768+0000"
}
],
"agent_state": {}
}
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
