So I did an export of the PG using ceph-objectstore-tool in hopes that I
could push ceph to forget about the rest of the data there. It was a
successful export but we’ll see how it goes importing it. I tried on one
osd already to import but got the message the PG already exists, am I doing
something wrong? Do I have to remove all fragments of the PG and force it
to go NOENT before trying an import?
On Wednesday, December 8, 2021, Eric Alba <noman.wonder@xxxxxxxxx> wrote:
> I've been trying to get ceph to force the PG to a good state but it
> continues to give me a single PG peering. This is a rook-ceph cluster on
> VMs (hosts went out for a brief period) and I can't figure out how to get
> this 1GB or so of data to become available to the client. This occurred
> during a cluster expansion. This gave the added joy of clock skew. Another
> 3 disks were being added to extend the cluster storage. This is a 3 node
> cluster with 6 200GB Disks each node.
>
> cluster:
> id: 31689324-f5ba-4aa4-8244-aa09a1119dc3
> health: HEALTH_OK
>
> services:
> mon: 3 daemons, quorum a,c,d (age 41m)
> mgr: a(active, since 2h)
> osd: 18 osds: 18 up (since 17m), 18 in (since 2h); 1 remapped pgs
>
> data:
> pools: 2 pools, 768 pgs
> objects: 184.38k objects, 707 GiB
> usage: 1.4 TiB used, 2.1 TiB / 3.5 TiB avail
> pgs: 0.130% pgs not active
> 767 active+clean
> 1 peering
>
> io:
> client: 14 KiB/s wr, 0 op/s rd, 1 op/s wr
>
> progress:
> Global Recovery Event (2h)
> [===========================.] (remaining: 9s)
>
> PG map for problematic PG
>
> [root@rook-ceph-tools-5b54fb98c-kjt5t /]# ceph pg map 2.11c
> osdmap e11328 pg 2.11c (2.11c) -> up [17,12] acting [4]
>
> Don't see OSD 4 in pg dump:
> [root@rook-ceph-tools-5b54fb98c-kjt5t /]# ceph pg dump 2> /dev/null |
> egrep '2\.11c|MISS'
> PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES
> OMAP_BYTES* OMAP_KEYS* LOG DISK_LOG STATE STATE_STAMP
> VERSION REPORTED UP UP_PRIMARY
> ACTING ACTING_PRIMARY LAST_SCRUB SCN
> 2.11c 249 0 0 0 0
> 1023758336 0 0 5352 5352 peering
> 2021-12-08T22:44:36.691204+0000 3669'8643974 11329:7414
> [17,12] 17 [17,12] 17 3162'8635517 200
>
>
>
> ceph pg 2.11c query
> {
> "snap_trimq": "[]",
> "snap_trimq_len": 0,
> "state": "unknown",
> "epoch": 12191,
> "up": [
> 17,
> 12
> ],
> "acting": [
> 17,
> 12
> ],
> "info": {
> "pgid": "2.11c",
> "last_update": "3669'8643974",
> "last_complete": "3669'8643974",
> "log_tail": "3174'8638622",
> "last_user_version": 8643974,
> "last_backfill": "2:38e2dedc:::rbd_data.1d091ec6540582.
> 000000000000158e:head",
> "purged_snaps": [],
> "history": {
> "epoch_created": 3286,
> "epoch_pool_created": 13,
> "last_epoch_started": 3725,
> "last_interval_started": 3724,
> "last_epoch_clean": 3544,
> "last_interval_clean": 3543,
> "last_epoch_split": 3286,
> "last_epoch_marked_full": 0,
> "same_up_since": 9923,
> "same_interval_since": 12191,
> "same_primary_since": 12191,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "3669'8643974",
> "reported_seq": 8277,
> "reported_epoch": 12191,
> "state": "peering",
> "last_fresh": "2021-12-08T22:59:01.450813+0000",
> "last_change": "2021-12-08T22:59:01.450813+0000",
> "last_active": "0.000000",
> "last_peered": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_became_peered": "0.000000",
> "last_unstale": "2021-12-08T22:59:01.450813+0000",
> "last_undegraded": "2021-12-08T22:59:01.450813+0000",
> "last_fullsized": "2021-12-08T22:59:01.450813+0000",
> "mapping_epoch": 12191,
> "log_start": "3174'8638622",
> "ondisk_log_start": "3174'8638622",
> "created": 3286,
> "last_epoch_clean": 3544,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "log_size": 5352,
> "ondisk_log_size": 5352,
> "stats_invalid": false,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 1023758336,
> "num_objects": 249,
> "num_object_clones": 0,
> "num_object_copies": 498,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 249,
> "num_whiteouts": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 0,
> "num_omap_keys": 0,
> "num_objects_repaired": 0
> },
> "up": [
> 17,
> 12
> ],
> "acting": [
> 17,
> 12
> ],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [
> 0,
> 1,
> 4,
> 6,
> 7,
> 12
> ],
> "up_primary": 17,
> "acting_primary": 17,
> "purged_snaps": []
> },
> "empty": 0,
> "dne": 0,
> "incomplete": 1,
> "last_epoch_started": 3565,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> },
> "peer_info": [
> {
> "peer": "0",
> "pgid": "2.11c",
> "last_update": "0'0",
> "last_complete": "0'0",
> "log_tail": "0'0",
> "last_user_version": 0,
> "last_backfill": "MAX",
> "purged_snaps": [],
> "history": {
> "epoch_created": 3286,
> "epoch_pool_created": 13,
> "last_epoch_started": 3725,
> "last_interval_started": 3724,
> "last_epoch_clean": 3544,
> "last_interval_clean": 3543,
> "last_epoch_split": 3286,
> "last_epoch_marked_full": 0,
> "same_up_since": 9923,
> "same_interval_since": 12191,
> "same_primary_since": 12191,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "0'0",
> "reported_seq": 2,
> "reported_epoch": 5384,
> "state": "peering",
> "last_fresh": "2021-12-08T20:54:53.732167+0000",
> "last_change": "2021-12-08T20:54:53.732009+0000",
> "last_active": "0.000000",
> "last_peered": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_became_peered": "0.000000",
> "last_unstale": "2021-12-08T20:54:53.732167+0000",
> "last_undegraded": "2021-12-08T20:54:53.732167+0000",
> "last_fullsized": "2021-12-08T20:54:53.732167+0000",
> "mapping_epoch": 12191,
> "log_start": "0'0",
> "ondisk_log_start": "0'0",
> "created": 3286,
> "last_epoch_clean": 3544,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "log_size": 0,
> "ondisk_log_size": 0,
> "stats_invalid": false,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 0,
> "num_objects": 0,
> "num_object_clones": 0,
> "num_object_copies": 0,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 0,
> "num_whiteouts": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 0,
> "num_omap_keys": 0,
> "num_objects_repaired": 0
> },
> "up": [
> 17,
> 12
> ],
> "acting": [
> 17,
> 12
> ],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [
> 1,
> 6,
> 7,
> 12
> ],
> "up_primary": 17,
> "acting_primary": 17,
> "purged_snaps": []
> },
> "empty": 1,
> "dne": 0,
> "incomplete": 0,
> "last_epoch_started": 0,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> },
> {
> "peer": "1",
> "pgid": "2.11c",
> "last_update": "0'0",
> "last_complete": "0'0",
> "log_tail": "0'0",
> "last_user_version": 0,
> "last_backfill": "MAX",
> "purged_snaps": [],
> "history": {
> "epoch_created": 0,
> "epoch_pool_created": 0,
> "last_epoch_started": 0,
> "last_interval_started": 0,
> "last_epoch_clean": 0,
> "last_interval_clean": 0,
> "last_epoch_split": 0,
> "last_epoch_marked_full": 0,
> "same_up_since": 0,
> "same_interval_since": 0,
> "same_primary_since": 0,
> "last_scrub": "0'0",
> "last_scrub_stamp": "0.000000",
> "last_deep_scrub": "0'0",
> "last_deep_scrub_stamp": "0.000000",
> "last_clean_scrub_stamp": "0.000000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "0'0",
> "reported_seq": 0,
> "reported_epoch": 0,
> "state": "unknown",
> "last_fresh": "0.000000",
> "last_change": "0.000000",
> "last_active": "0.000000",
> "last_peered": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_became_peered": "0.000000",
> "last_unstale": "0.000000",
> "last_undegraded": "0.000000",
> "last_fullsized": "0.000000",
> "mapping_epoch": 0,
> "log_start": "0'0",
> "ondisk_log_start": "0'0",
> "created": 0,
> "last_epoch_clean": 0,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "0'0",
> "last_scrub_stamp": "0.000000",
> "last_deep_scrub": "0'0",
> "last_deep_scrub_stamp": "0.000000",
> "last_clean_scrub_stamp": "0.000000",
> "log_size": 0,
> "ondisk_log_size": 0,
> "stats_invalid": false,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 0,
> "num_objects": 0,
> "num_object_clones": 0,
> "num_object_copies": 0,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 0,
> "num_whiteouts": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 0,
> "num_omap_keys": 0,
> "num_objects_repaired": 0
> },
> "up": [],
> "acting": [],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [],
> "up_primary": -1,
> "acting_primary": -1,
> "purged_snaps": []
> },
> "empty": 1,
> "dne": 1,
> "incomplete": 0,
> "last_epoch_started": 0,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> },
> {
> "peer": "4",
> "pgid": "2.11c",
> "last_update": "3425'8643930",
> "last_complete": "3425'8643930",
> "log_tail": "3174'8638622",
> "last_user_version": 8643930,
> "last_backfill": "MAX",
> "purged_snaps": [],
> "history": {
> "epoch_created": 3286,
> "epoch_pool_created": 13,
> "last_epoch_started": 3725,
> "last_interval_started": 3724,
> "last_epoch_clean": 3544,
> "last_interval_clean": 3543,
> "last_epoch_split": 3286,
> "last_epoch_marked_full": 0,
> "same_up_since": 9923,
> "same_interval_since": 12191,
> "same_primary_since": 12191,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "3425'8643930",
> "reported_seq": 11928359,
> "reported_epoch": 12190,
> "state": "remapped+peering",
> "last_fresh": "2021-12-08T22:59:00.436034+0000",
> "last_change": "2021-12-08T22:59:00.436034+0000",
> "last_active": "2021-12-07T20:45:51.712823+0000",
> "last_peered": "2021-12-07T20:22:45.680619+0000",
> "last_clean": "2021-12-07T20:17:36.452233+0000",
> "last_became_active": "2021-12-07T20:17:39.677192+0000",
> "last_became_peered": "2021-12-07T20:17:39.677192+0000",
> "last_unstale": "2021-12-08T22:59:00.436034+0000",
> "last_undegraded": "2021-12-08T22:59:00.436034+0000",
> "last_fullsized": "2021-12-08T22:59:00.436034+0000",
> "mapping_epoch": 12191,
> "log_start": "3174'8638622",
> "ondisk_log_start": "3174'8638622",
> "created": 3286,
> "last_epoch_clean": 3544,
> "parent": "0.0",
> "parent_split_bits": 9,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "log_size": 5308,
> "ondisk_log_size": 5308,
> "stats_invalid": true,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 1466349568,
> "num_objects": 358,
> "num_object_clones": 0,
> "num_object_copies": 716,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 358,
> "num_whiteouts": 0,
> "num_read": 442291,
> "num_read_kb": 6946464,
> "num_write": 1084007,
> "num_write_kb": 19438544,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 711,
> "num_bytes_recovered": 2800112640,
> "num_keys_recovered": 5,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 132,
> "num_omap_keys": 4,
> "num_objects_repaired": 0
> },
> "up": [
> 17,
> 12
> ],
> "acting": [
> 17,
> 12
> ],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [
> 0,
> 1,
> 6,
> 7,
> 12,
> 17
> ],
> "up_primary": 17,
> "acting_primary": 17,
> "purged_snaps": []
> },
> "empty": 0,
> "dne": 0,
> "incomplete": 0,
> "last_epoch_started": 3384,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> },
> {
> "peer": "6",
> "pgid": "2.11c",
> "last_update": "3669'8643974",
> "last_complete": "3669'8643974",
> "log_tail": "3174'8638622",
> "last_user_version": 0,
> "last_backfill": "MIN",
> "purged_snaps": [],
> "history": {
> "epoch_created": 3286,
> "epoch_pool_created": 13,
> "last_epoch_started": 3725,
> "last_interval_started": 3724,
> "last_epoch_clean": 3544,
> "last_interval_clean": 3543,
> "last_epoch_split": 3286,
> "last_epoch_marked_full": 0,
> "same_up_since": 9923,
> "same_interval_since": 12191,
> "same_primary_since": 12191,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "0'0",
> "reported_seq": 0,
> "reported_epoch": 0,
> "state": "unknown",
> "last_fresh": "0.000000",
> "last_change": "0.000000",
> "last_active": "0.000000",
> "last_peered": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_became_peered": "0.000000",
> "last_unstale": "0.000000",
> "last_undegraded": "0.000000",
> "last_fullsized": "0.000000",
> "mapping_epoch": 12191,
> "log_start": "0'0",
> "ondisk_log_start": "0'0",
> "created": 0,
> "last_epoch_clean": 0,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "0'0",
> "last_scrub_stamp": "0.000000",
> "last_deep_scrub": "0'0",
> "last_deep_scrub_stamp": "0.000000",
> "last_clean_scrub_stamp": "0.000000",
> "log_size": 0,
> "ondisk_log_size": 0,
> "stats_invalid": false,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 0,
> "num_objects": 0,
> "num_object_clones": 0,
> "num_object_copies": 0,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 0,
> "num_whiteouts": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 0,
> "num_omap_keys": 0,
> "num_objects_repaired": 0
> },
> "up": [
> 17,
> 12
> ],
> "acting": [
> 17,
> 12
> ],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [],
> "up_primary": 17,
> "acting_primary": 17,
> "purged_snaps": []
> },
> "empty": 0,
> "dne": 0,
> "incomplete": 1,
> "last_epoch_started": 3725,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> },
> {
> "peer": "7",
> "pgid": "2.11c",
> "last_update": "0'0",
> "last_complete": "0'0",
> "log_tail": "0'0",
> "last_user_version": 0,
> "last_backfill": "MAX",
> "purged_snaps": [],
> "history": {
> "epoch_created": 0,
> "epoch_pool_created": 0,
> "last_epoch_started": 0,
> "last_interval_started": 0,
> "last_epoch_clean": 0,
> "last_interval_clean": 0,
> "last_epoch_split": 0,
> "last_epoch_marked_full": 0,
> "same_up_since": 0,
> "same_interval_since": 0,
> "same_primary_since": 0,
> "last_scrub": "0'0",
> "last_scrub_stamp": "0.000000",
> "last_deep_scrub": "0'0",
> "last_deep_scrub_stamp": "0.000000",
> "last_clean_scrub_stamp": "0.000000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "0'0",
> "reported_seq": 0,
> "reported_epoch": 0,
> "state": "unknown",
> "last_fresh": "0.000000",
> "last_change": "0.000000",
> "last_active": "0.000000",
> "last_peered": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_became_peered": "0.000000",
> "last_unstale": "0.000000",
> "last_undegraded": "0.000000",
> "last_fullsized": "0.000000",
> "mapping_epoch": 0,
> "log_start": "0'0",
> "ondisk_log_start": "0'0",
> "created": 0,
> "last_epoch_clean": 0,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "0'0",
> "last_scrub_stamp": "0.000000",
> "last_deep_scrub": "0'0",
> "last_deep_scrub_stamp": "0.000000",
> "last_clean_scrub_stamp": "0.000000",
> "log_size": 0,
> "ondisk_log_size": 0,
> "stats_invalid": false,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 0,
> "num_objects": 0,
> "num_object_clones": 0,
> "num_object_copies": 0,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 0,
> "num_whiteouts": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 0,
> "num_omap_keys": 0,
> "num_objects_repaired": 0
> },
> "up": [],
> "acting": [],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [],
> "up_primary": -1,
> "acting_primary": -1,
> "purged_snaps": []
> },
> "empty": 1,
> "dne": 1,
> "incomplete": 0,
> "last_epoch_started": 0,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> },
> {
> "peer": "12",
> "pgid": "2.11c",
> "last_update": "0'0",
> "last_complete": "0'0",
> "log_tail": "0'0",
> "last_user_version": 0,
> "last_backfill": "MAX",
> "purged_snaps": [],
> "history": {
> "epoch_created": 3286,
> "epoch_pool_created": 13,
> "last_epoch_started": 3725,
> "last_interval_started": 3724,
> "last_epoch_clean": 3544,
> "last_interval_clean": 3543,
> "last_epoch_split": 3286,
> "last_epoch_marked_full": 0,
> "same_up_since": 9923,
> "same_interval_since": 12191,
> "same_primary_since": 12191,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "prior_readable_until_ub": 0
> },
> "stats": {
> "version": "0'0",
> "reported_seq": 386,
> "reported_epoch": 9872,
> "state": "down",
> "last_fresh": "2021-12-08T22:10:55.852651+0000",
> "last_change": "2021-12-08T22:10:54.848737+0000",
> "last_active": "0.000000",
> "last_peered": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_became_peered": "0.000000",
> "last_unstale": "2021-12-08T22:10:55.852651+0000",
> "last_undegraded": "2021-12-08T22:10:55.852651+0000",
> "last_fullsized": "2021-12-08T22:10:55.852651+0000",
> "mapping_epoch": 12191,
> "log_start": "0'0",
> "ondisk_log_start": "0'0",
> "created": 3286,
> "last_epoch_clean": 3544,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "3162'8635517",
> "last_scrub_stamp": "2021-12-06T22:16:26.082945+0000",
> "last_deep_scrub": "3122'8608768",
> "last_deep_scrub_stamp": "2021-12-04T02:35:48.882906+
> 0000",
> "last_clean_scrub_stamp": "2021-12-06T22:16:26.082945+
> 0000",
> "log_size": 0,
> "ondisk_log_size": 0,
> "stats_invalid": false,
> "dirty_stats_invalid": false,
> "omap_stats_invalid": false,
> "hitset_stats_invalid": false,
> "hitset_bytes_stats_invalid": false,
> "pin_stats_invalid": false,
> "manifest_stats_invalid": false,
> "snaptrimq_len": 0,
> "stat_sum": {
> "num_bytes": 0,
> "num_objects": 0,
> "num_object_clones": 0,
> "num_object_copies": 0,
> "num_objects_missing_on_primary": 0,
> "num_objects_missing": 0,
> "num_objects_degraded": 0,
> "num_objects_misplaced": 0,
> "num_objects_unfound": 0,
> "num_objects_dirty": 0,
> "num_whiteouts": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0,
> "num_objects_omap": 0,
> "num_objects_hit_set_archive": 0,
> "num_bytes_hit_set_archive": 0,
> "num_flush": 0,
> "num_flush_kb": 0,
> "num_evict": 0,
> "num_evict_kb": 0,
> "num_promote": 0,
> "num_flush_mode_high": 0,
> "num_flush_mode_low": 0,
> "num_evict_mode_some": 0,
> "num_evict_mode_full": 0,
> "num_objects_pinned": 0,
> "num_legacy_snapsets": 0,
> "num_large_omap_objects": 0,
> "num_objects_manifest": 0,
> "num_omap_bytes": 0,
> "num_omap_keys": 0,
> "num_objects_repaired": 0
> },
> "up": [
> 17,
> 12
> ],
> "acting": [
> 17,
> 12
> ],
> "avail_no_missing": [],
> "object_location_counts": [],
> "blocked_by": [
> 17
> ],
> "up_primary": 17,
> "acting_primary": 17,
> "purged_snaps": []
> },
> "empty": 1,
> "dne": 0,
> "incomplete": 0,
> "last_epoch_started": 0,
> "hit_set_history": {
> "current_last_update": "0'0",
> "history": []
> }
> }
> ],
> "recovery_state": [
> {
> "name": "Started/Primary/WaitActingChange",
> "enter_time": "2021-12-08T22:59:01.462157+0000",
> "comment": "waiting for pg acting set to change"
> },
> {
> "name": "Started",
> "enter_time": "2021-12-08T22:59:01.450768+0000"
> }
> ],
> "agent_state": {}
> }
>
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
