This issue happened a week ago, so I don't have output from pg query.
Now on the test cluster I am observing similiar problems. Output from
query in attachment.
data:
pools: 5 pools, 32800 pgs
objects: 11.53E objects, 62.2GiB
usage: 176GiB used, 1.05TiB / 1.23TiB avail
pgs: 30.899% pgs not active
20193 active+clean
7573 activating+degraded
2525 activating
2460 active+recovery_wait+degraded
14 remapped+peering
11 down
5 activating+degraded+remapped
4 activating+remapped
3 active+recovery_wait+degraded+remapped
2 stale+active+clean
2 peering
2 active+clean+remapped
1 active+undersized+degraded
1 activating+undersized+degraded
1 active+recovery_wait+undersized+degraded
1 active+recovery_wait
1 active+recovering
1 active+recovering+degraded
pool 5 'test' erasure size 6 min_size 4 crush_rule 1 object_hash
rjenkins pg_num 32768 pgp_num 32768 last_change 153 lfor 0/150 flags
hashpspool stripe_width 16384 application rbd
It looks that cluster is blocked by something... This cluster is 12.2.11
Best Regards,
Rafał Wądołowski
On 25.03.2019 10:56, Sage Weil wrote:
> On Mon, 25 Mar 2019, Rafał Wądołowski wrote:
>> Hi,
>>
>> On one of our cluster (3400 OSD, ~25PB, 12.2.4), we incremented pg_num &
>> pgp_num on one pool (EC 4+2) from 32k to 64k. After that cluster started
>> to be instable for one hour, pgs were inactive (some activating, some
>> peering).
>>
>> Any idea what bottlenecks we hit? Any ideas what should I change in
>> configuration of ceph/os ?
> Could be lots of things.
>
> What does 'ceph tell <pgid> query' show for one of the activating or
> peering pgs?
>
> Note that you're moving ~half of hte data around in yoru cluster with that
> change, so you will see each of those PGs cycle through backfill ->
> peering -> activating -> active in the course of it moving.
>
> sage
{
"state": "activating+degraded",
"snap_trimq": "[]",
"snap_trimq_len": 0,
"epoch": 4033,
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"actingbackfill": [
"22(4)",
"27(2)",
"32(5)",
"39(0)",
"40(3)",
"45(1)"
],
"info": {
"pgid": "5.7fe8s0",
"last_update": "113'30",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "3924",
"reported_epoch": "4033",
"state": "activating+degraded",
"last_fresh": "2019-03-25 11:05:48.377218",
"last_change": "2019-03-25 08:57:47.257521",
"last_active": "2019-03-25 08:57:43.070460",
"last_peered": "2019-03-25 08:57:41.730206",
"last_clean": "2019-03-25 08:47:57.808495",
"last_became_active": "2019-03-25 08:55:03.323331",
"last_became_peered": "2019-03-25 08:55:03.323331",
"last_unstale": "2019-03-25 11:05:48.377218",
"last_undegraded": "2019-03-25 08:57:47.204881",
"last_fullsized": "2019-03-25 11:05:48.377218",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 150,
"last_epoch_clean": 152,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2097152,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 1,
"num_objects_missing": 0,
"num_objects_degraded": 6,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [
1
],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 473,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "5(0)",
"pgid": "5.7fe8s0",
"last_update": "113'30",
"last_complete": "113'30",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "71",
"reported_epoch": "152",
"state": "active+clean",
"last_fresh": "2019-03-25 08:47:57.808495",
"last_change": "2019-03-25 08:47:57.428794",
"last_active": "2019-03-25 08:47:57.808495",
"last_peered": "2019-03-25 08:47:57.808495",
"last_clean": "2019-03-25 08:47:57.808495",
"last_became_active": "2019-03-25 08:47:57.428078",
"last_became_peered": "2019-03-25 08:47:57.428078",
"last_unstale": "2019-03-25 08:47:57.808495",
"last_undegraded": "2019-03-25 08:47:57.808495",
"last_fullsized": "2019-03-25 08:47:57.808495",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 150,
"last_epoch_clean": 152,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2097152,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 152,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "7(3)",
"pgid": "5.7fe8s3",
"last_update": "113'30",
"last_complete": "113'30",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "6",
"reported_epoch": "123",
"state": "peering",
"last_fresh": "2019-03-25 08:14:01.252335",
"last_change": "2019-03-25 08:02:32.845180",
"last_active": "2019-03-25 08:02:32.845180",
"last_peered": "2019-03-25 07:59:38.582473",
"last_clean": "2019-03-25 07:59:38.582473",
"last_became_active": "2019-03-25 07:44:15.844378",
"last_became_peered": "2019-03-25 07:44:15.844378",
"last_unstale": "2019-03-25 08:14:01.252335",
"last_undegraded": "2019-03-25 08:14:01.252335",
"last_fullsized": "2019-03-25 08:14:01.252335",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 119,
"last_epoch_clean": 121,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "106'22",
"last_scrub_stamp": "2019-03-24 22:32:36.374292",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-24 22:32:36.374292",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1572864,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 3,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 3,
"num_write_kb": 5632,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2,
"num_bytes_recovered": 5242880,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 152,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "15(5)",
"pgid": "5.7fe8s5",
"last_update": "113'30",
"last_complete": "113'30",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "6",
"reported_epoch": "123",
"state": "peering",
"last_fresh": "2019-03-25 08:14:01.252335",
"last_change": "2019-03-25 08:02:32.845180",
"last_active": "2019-03-25 08:02:32.845180",
"last_peered": "2019-03-25 07:59:38.582473",
"last_clean": "2019-03-25 07:59:38.582473",
"last_became_active": "2019-03-25 07:44:15.844378",
"last_became_peered": "2019-03-25 07:44:15.844378",
"last_unstale": "2019-03-25 08:14:01.252335",
"last_undegraded": "2019-03-25 08:14:01.252335",
"last_fullsized": "2019-03-25 08:14:01.252335",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 119,
"last_epoch_clean": 121,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "106'22",
"last_scrub_stamp": "2019-03-24 22:32:36.374292",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-24 22:32:36.374292",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1572864,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 3,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 3,
"num_write_kb": 5632,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2,
"num_bytes_recovered": 5242880,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 152,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "22(4)",
"pgid": "5.7fe8s4",
"last_update": "113'30",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "7",
"reported_epoch": "156",
"state": "peering",
"last_fresh": "2019-03-25 08:48:25.217905",
"last_change": "2019-03-25 08:48:25.217879",
"last_active": "2019-03-25 08:48:25.217879",
"last_peered": "2019-03-25 08:47:57.808495",
"last_clean": "2019-03-25 08:47:57.808495",
"last_became_active": "2019-03-25 08:47:57.428078",
"last_became_peered": "2019-03-25 08:47:57.428078",
"last_unstale": "2019-03-25 08:48:25.217905",
"last_undegraded": "2019-03-25 08:48:25.217905",
"last_fullsized": "2019-03-25 08:48:25.217905",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 150,
"last_epoch_clean": 152,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2097152,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 343,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "27(1)",
"pgid": "5.7fe8s1",
"last_update": "113'30",
"last_complete": "113'30",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "6",
"reported_epoch": "123",
"state": "peering",
"last_fresh": "2019-03-25 08:14:01.252335",
"last_change": "2019-03-25 08:02:32.845180",
"last_active": "2019-03-25 08:02:32.845180",
"last_peered": "2019-03-25 07:59:38.582473",
"last_clean": "2019-03-25 07:59:38.582473",
"last_became_active": "2019-03-25 07:44:15.844378",
"last_became_peered": "2019-03-25 07:44:15.844378",
"last_unstale": "2019-03-25 08:14:01.252335",
"last_undegraded": "2019-03-25 08:14:01.252335",
"last_fullsized": "2019-03-25 08:14:01.252335",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 119,
"last_epoch_clean": 121,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "106'22",
"last_scrub_stamp": "2019-03-24 22:32:36.374292",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-24 22:32:36.374292",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1572864,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 3,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 3,
"num_write_kb": 5632,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2,
"num_bytes_recovered": 5242880,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 343,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "27(2)",
"pgid": "5.7fe8s2",
"last_update": "113'30",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "7",
"reported_epoch": "156",
"state": "peering",
"last_fresh": "2019-03-25 08:48:25.217905",
"last_change": "2019-03-25 08:48:25.217879",
"last_active": "2019-03-25 08:48:25.217879",
"last_peered": "2019-03-25 08:47:57.808495",
"last_clean": "2019-03-25 08:47:57.808495",
"last_became_active": "2019-03-25 08:47:57.428078",
"last_became_peered": "2019-03-25 08:47:57.428078",
"last_unstale": "2019-03-25 08:48:25.217905",
"last_undegraded": "2019-03-25 08:48:25.217905",
"last_fullsized": "2019-03-25 08:48:25.217905",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 150,
"last_epoch_clean": 152,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2097152,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 343,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "32(5)",
"pgid": "5.7fe8s5",
"last_update": "113'30",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "7",
"reported_epoch": "156",
"state": "peering",
"last_fresh": "2019-03-25 08:48:25.217905",
"last_change": "2019-03-25 08:48:25.217879",
"last_active": "2019-03-25 08:48:25.217879",
"last_peered": "2019-03-25 08:47:57.808495",
"last_clean": "2019-03-25 08:47:57.808495",
"last_became_active": "2019-03-25 08:47:57.428078",
"last_became_peered": "2019-03-25 08:47:57.428078",
"last_unstale": "2019-03-25 08:48:25.217905",
"last_undegraded": "2019-03-25 08:48:25.217905",
"last_fullsized": "2019-03-25 08:48:25.217905",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 150,
"last_epoch_clean": 152,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2097152,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 343,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "40(3)",
"pgid": "5.7fe8s3",
"last_update": "113'30",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "8",
"reported_epoch": "266",
"state": "unknown",
"last_fresh": "2019-03-25 08:54:17.269781",
"last_change": "2019-03-25 08:54:17.269781",
"last_active": "2019-03-25 08:48:25.217879",
"last_peered": "2019-03-25 08:47:57.808495",
"last_clean": "2019-03-25 08:47:57.808495",
"last_became_active": "2019-03-25 08:47:57.428078",
"last_became_peered": "2019-03-25 08:47:57.428078",
"last_unstale": "2019-03-25 08:54:17.269781",
"last_undegraded": "2019-03-25 08:54:17.269781",
"last_fullsized": "2019-03-25 08:54:17.269781",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 150,
"last_epoch_clean": 152,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 2097152,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 6,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 343,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "45(1)",
"pgid": "5.7fe8s1",
"last_update": "113'30",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [],
"history": {
"epoch_created": 0,
"epoch_pool_created": 0,
"last_epoch_started": 0,
"last_interval_started": 0,
"last_epoch_clean": 0,
"last_interval_clean": 0,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 1,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1
},
"empty": 0,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "50(2)",
"pgid": "5.7fe8s2",
"last_update": "113'30",
"last_complete": "113'30",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "6",
"reported_epoch": "123",
"state": "peering",
"last_fresh": "2019-03-25 08:14:01.252335",
"last_change": "2019-03-25 08:02:32.845180",
"last_active": "2019-03-25 08:02:32.845180",
"last_peered": "2019-03-25 07:59:38.582473",
"last_clean": "2019-03-25 07:59:38.582473",
"last_became_active": "2019-03-25 07:44:15.844378",
"last_became_peered": "2019-03-25 07:44:15.844378",
"last_unstale": "2019-03-25 08:14:01.252335",
"last_undegraded": "2019-03-25 08:14:01.252335",
"last_fullsized": "2019-03-25 08:14:01.252335",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 119,
"last_epoch_clean": 121,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "106'22",
"last_scrub_stamp": "2019-03-24 22:32:36.374292",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-24 22:32:36.374292",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1572864,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 3,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 3,
"num_write_kb": 5632,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2,
"num_bytes_recovered": 5242880,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 152,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "53(4)",
"pgid": "5.7fe8s4",
"last_update": "113'30",
"last_complete": "113'30",
"log_tail": "0'0",
"last_user_version": 30,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [],
"history": {
"epoch_created": 150,
"epoch_pool_created": 104,
"last_epoch_started": 343,
"last_interval_started": 341,
"last_epoch_clean": 152,
"last_interval_clean": 150,
"last_epoch_split": 150,
"last_epoch_marked_full": 0,
"same_up_since": 469,
"same_interval_since": 472,
"same_primary_since": 267,
"last_scrub": "113'30",
"last_scrub_stamp": "2019-03-25 08:37:43.117291",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-25 08:37:43.117291"
},
"stats": {
"version": "113'30",
"reported_seq": "6",
"reported_epoch": "123",
"state": "peering",
"last_fresh": "2019-03-25 08:14:01.252335",
"last_change": "2019-03-25 08:02:32.845180",
"last_active": "2019-03-25 08:02:32.845180",
"last_peered": "2019-03-25 07:59:38.582473",
"last_clean": "2019-03-25 07:59:38.582473",
"last_became_active": "2019-03-25 07:44:15.844378",
"last_became_peered": "2019-03-25 07:44:15.844378",
"last_unstale": "2019-03-25 08:14:01.252335",
"last_undegraded": "2019-03-25 08:14:01.252335",
"last_fullsized": "2019-03-25 08:14:01.252335",
"mapping_epoch": 472,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 119,
"last_epoch_clean": 121,
"parent": "0.0",
"parent_split_bits": 15,
"last_scrub": "106'22",
"last_scrub_stamp": "2019-03-24 22:32:36.374292",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2019-03-22 08:39:51.364371",
"last_clean_scrub_stamp": "2019-03-24 22:32:36.374292",
"log_size": 30,
"ondisk_log_size": 30,
"stats_invalid": true,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"snaptrimq_len": 0,
"stat_sum": {
"num_bytes": 1572864,
"num_objects": 1,
"num_object_clones": 0,
"num_object_copies": 3,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 3,
"num_write_kb": 5632,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 2,
"num_bytes_recovered": 5242880,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0,
"num_large_omap_objects": 0
},
"up": [
39,
45,
27,
40,
22,
32
],
"acting": [
39,
45,
27,
40,
22,
32
],
"blocked_by": [],
"up_primary": 39,
"acting_primary": 39
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 152,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2019-03-25 08:57:47.204627",
"might_have_unfound": [
{
"osd": "5(0)",
"status": "already probed"
},
{
"osd": "7(3)",
"status": "already probed"
},
{
"osd": "15(5)",
"status": "already probed"
},
{
"osd": "22(4)",
"status": "already probed"
},
{
"osd": "27(1)",
"status": "already probed"
},
{
"osd": "27(2)",
"status": "already probed"
},
{
"osd": "32(5)",
"status": "already probed"
},
{
"osd": "40(3)",
"status": "already probed"
},
{
"osd": "45(1)",
"status": "already probed"
},
{
"osd": "50(2)",
"status": "already probed"
},
{
"osd": "53(4)",
"status": "already probed"
}
],
"recovery_progress": {
"backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"recovery_ops": [],
"read_ops": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.max_end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2019-03-25 08:57:46.241917"
}
],
"agent_state": {}
}
_______________________________________________
Ceph-large mailing list
Ceph-large@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-large-ceph.com
