Hello,
Env:-
5 node, EC 4+1 bluestore kraken v11.2.0 , RHEL7.2
As part of our resillency testing with kraken bluestore, we face more PG's were in incomplete+remapped state. We tried to repair each PG using "ceph pg repair <pgid>" still no luck. Then we planned to remove incomplete PG's using below procedure.
#ceph health detail | grep 1.e4b
pg 1.e4b is remapped+incomplete, acting [2147483647,66,15,73,2147483647] (reducing pool cdvr_ec min_size from 4 may help; search ceph.com/docs for 'incomplete')
Here we shutdown the OSD's 66,15 and 73 then proceeded with below operation.
#ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-135 --op list-pgs
#ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-135 --pgid 1.e4b --op remove
Please confirm that we are following the correct procedure to removal of PG's
#ceph pg stat
v2724830: 4096 pgs: 1 active+clean+scrubbing+deep+repair, 1 down+remapped, 21 remapped+incomplete, 4073 active+clean; 268 TB data, 371 TB used, 267 TB / 638 TB avail
# ceph -s
2017-03-29 18:23:44.288508 7f8c2b8e5700 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
2017-03-29 18:23:44.304692 7f8c2b8e5700 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
cluster bd8adcd0-c36d-4367-9efe-f48f5ab5f108
health HEALTH_ERR
22 pgs are stuck inactive for more than 300 seconds
1 pgs down
21 pgs incomplete
1 pgs repair
22 pgs stuck inactive
22 pgs stuck unclean
monmap e2: 5 mons at {au-adelaide=10.50.21.24:6789/0,au-brisbane=10.50.21.22:6789/0,au-canberra=10.50.21.23:6789/0,au-melbourne=10.50.21.21:6789/0,au-sydney=10.50.21.20:6789/0}
election epoch 172, quorum 0,1,2,3,4 au-sydney,au-melbourne,au-brisbane,au-canberra,au-adelaide
mgr active: au-brisbane
osdmap e6284: 118 osds: 117 up, 117 in; 22 remapped pgs
flags sortbitwise,require_jewel_osds,require_kraken_osds
pgmap v2724830: 4096 pgs, 1 pools, 268 TB data, 197 Mobjects
371 TB used, 267 TB / 638 TB avail
4073 active+clean
21 remapped+incomplete
1 down+remapped
1 active+clean+scrubbing+deep+repair
#ceph osd dump | grep pool
pool 1 'cdvr_ec' erasure size 5 min_size 4 crush_ruleset 1 object_hash rjenkins pg_num 4096 pgp_num 4096 last_change 456 flags hashpspool,nodeep-scrub stripe_width 65536
Can you please suggest is there any way to wipe out these incomplete PG's. Why ceph pg repair failed in this scenerio?
How to recover incomplete PG's to active state.
pg query for the affected PG ended with this error. Can you please explain what is meant by this ?
---
"15(2)",
"66(1)",
"73(3)",
"103(4)",
"113(0)"
],
"down_osds_we_would_probe": [
6
],
"peering_blocked_by": [],
"peering_blocked_by_detail": [
{
"detail": "peering_blocked_by_history_les_bound"
}
----
Attaching "ceph pg 1.e4b query > /tmp/1.e4b-pg.txt" file with this mail.
Thanks
{
"state": "remapped+incomplete",
"snap_trimq": "[]",
"epoch": 6284,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"info": {
"pgid": "1.e4bs1",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "0'0",
"reported_seq": "9794",
"reported_epoch": "6279",
"state": "remapped+incomplete",
"last_fresh": "2017-03-29 14:36:41.192157",
"last_change": "2017-03-29 13:48:34.668294",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2017-03-29 14:36:41.192157",
"last_undegraded": "2017-03-29 14:36:41.192157",
"last_fullsized": "2017-03-29 14:36:41.192157",
"mapping_epoch": 6270,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 435,
"last_epoch_clean": 1795,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [
6
],
"up_primary": 113,
"acting_primary": 66
},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "15(2)",
"pgid": "1.e4bs2",
"last_update": "3844'161662",
"last_complete": "3844'161662",
"log_tail": "2355'151656",
"last_user_version": 161662,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "3844'161662",
"reported_seq": "326110",
"reported_epoch": "6075",
"state": "peering",
"last_fresh": "2017-03-29 10:29:50.999580",
"last_change": "2017-03-29 10:29:49.634894",
"last_active": "2017-03-27 10:20:14.426063",
"last_peered": "2017-03-27 10:20:14.426063",
"last_clean": "2017-03-21 16:31:47.009315",
"last_became_active": "2017-03-27 09:54:42.509907",
"last_became_peered": "2017-03-27 09:54:42.509907",
"last_unstale": "2017-03-29 10:29:50.999580",
"last_undegraded": "2017-03-29 10:29:50.999580",
"last_fullsized": "2017-03-29 10:29:50.999580",
"mapping_epoch": 6270,
"log_start": "2355'151656",
"ondisk_log_start": "2355'151656",
"created": 435,
"last_epoch_clean": 1795,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953",
"log_size": 10006,
"ondisk_log_size": 10006,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 73013340821,
"num_objects": 50832,
"num_object_clones": 0,
"num_object_copies": 254160,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 50832,
"num_whiteouts": 0,
"num_read": 160530,
"num_read_kb": 77492988,
"num_write": 1429226,
"num_write_kb": 154928294,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 59773,
"num_bytes_recovered": 87096543043,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 3831,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "73(3)",
"pgid": "1.e4bs3",
"last_update": "3844'161662",
"last_complete": "3844'161662",
"log_tail": "2355'151656",
"last_user_version": 161662,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "3844'161662",
"reported_seq": "325574",
"reported_epoch": "6116",
"state": "peering",
"last_fresh": "2017-03-29 10:30:52.014622",
"last_change": "2017-03-29 10:30:51.148893",
"last_active": "2017-03-27 10:20:14.426063",
"last_peered": "2017-03-27 10:20:14.426063",
"last_clean": "2017-03-21 16:31:47.009315",
"last_became_active": "2017-03-27 09:54:42.509907",
"last_became_peered": "2017-03-27 09:54:42.509907",
"last_unstale": "2017-03-29 10:30:52.014622",
"last_undegraded": "2017-03-29 10:30:52.014622",
"last_fullsized": "2017-03-29 10:30:52.014622",
"mapping_epoch": 6270,
"log_start": "2355'151656",
"ondisk_log_start": "2355'151656",
"created": 435,
"last_epoch_clean": 1795,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953",
"log_size": 10006,
"ondisk_log_size": 10006,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 73013340821,
"num_objects": 50832,
"num_object_clones": 0,
"num_object_copies": 254160,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 50832,
"num_whiteouts": 0,
"num_read": 160530,
"num_read_kb": 77492988,
"num_write": 1429226,
"num_write_kb": 154928294,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 59773,
"num_bytes_recovered": 87096543043,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 3831,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "81(4)",
"pgid": "1.e4bs4",
"last_update": "3377'158777",
"last_complete": "3377'158777",
"log_tail": "2315'148754",
"last_user_version": 158776,
"last_backfill": "1:d2708b14:::%2fc11%2fvx0000041%2fpot%2fsoak15479%2fhls%2fvar947000%2f00005%2fseg2308.ts:head",
"last_backfill_bitwise": 1,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 1815,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 6270,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 2394252038,
"num_objects": 1661,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1661,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 4582,
"num_write_kb": 499934,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1815,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "103(4)",
"pgid": "1.e4bs4",
"last_update": "3844'161662",
"last_complete": "3844'161662",
"log_tail": "2431'155777",
"last_user_version": 161662,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "1815'142337",
"reported_seq": "302697",
"reported_epoch": "1815",
"state": "active+remapped+backfill_wait",
"last_fresh": "2017-03-21 16:31:57.558580",
"last_change": "2017-03-21 16:31:57.558580",
"last_active": "2017-03-21 16:31:57.558580",
"last_peered": "2017-03-21 16:31:57.558580",
"last_clean": "2017-03-21 16:31:47.009315",
"last_became_active": "2017-03-21 16:31:56.835627",
"last_became_peered": "2017-03-21 16:31:56.835627",
"last_unstale": "2017-03-21 16:31:57.558580",
"last_undegraded": "2017-03-21 16:31:57.558580",
"last_fullsized": "2017-03-21 16:31:57.558580",
"mapping_epoch": 6270,
"log_start": "1643'139252",
"ondisk_log_start": "1643'139252",
"created": 435,
"last_epoch_clean": 1795,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953",
"log_size": 3085,
"ondisk_log_size": 3085,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 3379,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "106(0)",
"pgid": "1.e4bs0",
"last_update": "3377'158777",
"last_complete": "3377'158777",
"log_tail": "2315'148754",
"last_user_version": 158776,
"last_backfill": "1:d2700429:::%2fc11%2fvx0000041%2fpot%2fchannel_sd_106%2fhls%2fperiod1%2fvar478000%2f00012%2fseg0226.ts:head",
"last_backfill_bitwise": 1,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 6270,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 65259876,
"num_objects": 46,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 46,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 109,
"num_write_kb": 13481,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1815,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "113(0)",
"pgid": "1.e4bs0",
"last_update": "3844'161662",
"last_complete": "3844'161662",
"log_tail": "2355'151656",
"last_user_version": 161662,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "3844'161662",
"reported_seq": "327240",
"reported_epoch": "6266",
"state": "inactive",
"last_fresh": "2017-03-29 13:48:32.487299",
"last_change": "2017-03-29 13:48:32.487299",
"last_active": "2017-03-27 10:20:14.426063",
"last_peered": "2017-03-27 10:20:14.426063",
"last_clean": "2017-03-21 16:31:47.009315",
"last_became_active": "2017-03-27 09:54:42.509907",
"last_became_peered": "2017-03-27 09:54:42.509907",
"last_unstale": "2017-03-29 13:48:32.487299",
"last_undegraded": "2017-03-29 13:48:32.487299",
"last_fullsized": "2017-03-29 13:48:32.487299",
"mapping_epoch": 6270,
"log_start": "2355'151656",
"ondisk_log_start": "2355'151656",
"created": 435,
"last_epoch_clean": 1795,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953",
"log_size": 10006,
"ondisk_log_size": 10006,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 73013340821,
"num_objects": 50832,
"num_object_clones": 0,
"num_object_copies": 254160,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 50832,
"num_whiteouts": 0,
"num_read": 160530,
"num_read_kb": 77492988,
"num_write": 1429226,
"num_write_kb": 154928294,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 59773,
"num_bytes_recovered": 87096543043,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 3831,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "118(1)",
"pgid": "1.e4bs1",
"last_update": "3377'158777",
"last_complete": "3377'158777",
"log_tail": "2315'148754",
"last_user_version": 158776,
"last_backfill": "1:d2708b14:::%2fc11%2fvx0000041%2fpot%2fsoak15479%2fhls%2fvar947000%2f00005%2fseg2308.ts:head",
"last_backfill_bitwise": 1,
"purged_snaps": "[]",
"history": {
"epoch_created": 435,
"last_epoch_started": 3831,
"last_epoch_clean": 1795,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 6270,
"same_interval_since": 6270,
"same_primary_since": 6267,
"last_scrub": "1643'139486",
"last_scrub_stamp": "2017-03-21 04:56:16.683953",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2017-02-21 10:33:50.012922",
"last_clean_scrub_stamp": "2017-03-21 04:56:16.683953"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 6270,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 2394252038,
"num_objects": 1661,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1661,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 4582,
"num_write_kb": 499934,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0
},
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"blocked_by": [],
"up_primary": 113,
"acting_primary": 66
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 1815,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Peering\/Incomplete",
"enter_time": "2017-03-29 13:48:34.668288",
"comment": "not enough complete instances of this PG"
},
{
"name": "Started\/Primary\/Peering",
"enter_time": "2017-03-29 13:48:34.635068",
"past_intervals": [
{
"first": 892,
"last": 906,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
106,
118,
15,
73,
81
],
"primary": 106,
"up_primary": 106
},
{
"first": 907,
"last": 1607,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
106,
118,
15,
73,
81
],
"primary": 106,
"up_primary": 113
},
{
"first": 1608,
"last": 1645,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 1646,
"last": 1653,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
2147483647
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 1654,
"last": 1654,
"maybe_went_rw": 0,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 1655,
"last": 1671,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 1672,
"last": 1691,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 106
},
{
"first": 1692,
"last": 1775,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 1776,
"last": 1793,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 106
},
{
"first": 1794,
"last": 1813,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 1814,
"last": 1818,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
103
],
"primary": 113,
"up_primary": 106
},
{
"first": 1819,
"last": 1839,
"maybe_went_rw": 1,
"up": [
2147483647,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 118
},
{
"first": 1840,
"last": 3301,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 106
},
{
"first": 3302,
"last": 3328,
"maybe_went_rw": 0,
"up": [
106,
118,
2147483647,
73,
81
],
"acting": [
113,
6,
2147483647,
73,
2147483647
],
"primary": 113,
"up_primary": 106
},
{
"first": 3329,
"last": 3329,
"maybe_went_rw": 0,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
2147483647,
73,
2147483647
],
"primary": 113,
"up_primary": 106
},
{
"first": 3330,
"last": 3366,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 106
},
{
"first": 3367,
"last": 3370,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3371,
"last": 3377,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
81
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 106
},
{
"first": 3378,
"last": 3716,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3717,
"last": 3771,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3772,
"last": 3793,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3794,
"last": 3823,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3824,
"last": 3824,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3825,
"last": 3828,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3829,
"last": 3829,
"maybe_went_rw": 0,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3830,
"last": 3844,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
113,
6,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3845,
"last": 3846,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3847,
"last": 3847,
"maybe_went_rw": 0,
"up": [
113,
36,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3848,
"last": 3850,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3851,
"last": 3868,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3869,
"last": 3889,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3890,
"last": 3910,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
2147483647
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3911,
"last": 3911,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3912,
"last": 3965,
"maybe_went_rw": 0,
"up": [
106,
118,
15,
73,
28
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 106
},
{
"first": 3966,
"last": 3971,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 3972,
"last": 4003,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
2147483647
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 4004,
"last": 4007,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 4008,
"last": 4036,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
2147483647,
103
],
"acting": [
113,
2147483647,
15,
2147483647,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 4037,
"last": 4045,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
2147483647,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 4046,
"last": 4064,
"maybe_went_rw": 0,
"up": [
113,
66,
2147483647,
73,
103
],
"acting": [
113,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 4065,
"last": 4082,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 4083,
"last": 4083,
"maybe_went_rw": 0,
"up": [
2147483647,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": -1,
"up_primary": 66
},
{
"first": 4084,
"last": 4090,
"maybe_went_rw": 1,
"up": [
2147483647,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 66
},
{
"first": 4091,
"last": 4133,
"maybe_went_rw": 1,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 4134,
"last": 4136,
"maybe_went_rw": 1,
"up": [
113,
36,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 4137,
"last": 4138,
"maybe_went_rw": 1,
"up": [
113,
6,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 4139,
"last": 4140,
"maybe_went_rw": 1,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 4141,
"last": 4141,
"maybe_went_rw": 0,
"up": [
113,
36,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 4142,
"last": 4145,
"maybe_went_rw": 1,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 4146,
"last": 4153,
"maybe_went_rw": 1,
"up": [
106,
118,
15,
73,
28
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 106
},
{
"first": 4154,
"last": 5751,
"maybe_went_rw": 1,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
103
],
"primary": 66,
"up_primary": 113
},
{
"first": 5752,
"last": 5765,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
2147483647
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5766,
"last": 5801,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5802,
"last": 5824,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
2147483647,
103
],
"acting": [
2147483647,
66,
15,
2147483647,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5825,
"last": 5834,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
2147483647,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5835,
"last": 5844,
"maybe_went_rw": 0,
"up": [
113,
66,
2147483647,
73,
103
],
"acting": [
2147483647,
66,
2147483647,
2147483647,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5845,
"last": 5872,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
2147483647,
2147483647,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5873,
"last": 5879,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
2147483647
],
"acting": [
2147483647,
66,
2147483647,
2147483647,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5880,
"last": 5894,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
2147483647,
2147483647,
2147483647
],
"primary": 66,
"up_primary": 113
},
{
"first": 5895,
"last": 5895,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": -1,
"up_primary": 113
},
{
"first": 5896,
"last": 5910,
"maybe_went_rw": 1,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 5911,
"last": 5922,
"maybe_went_rw": 1,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
73,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 5923,
"last": 5933,
"maybe_went_rw": 0,
"up": [
2147483647,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
103
],
"primary": 15,
"up_primary": 66
},
{
"first": 5934,
"last": 5981,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
103
],
"primary": 15,
"up_primary": 113
},
{
"first": 5982,
"last": 5986,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
103
],
"primary": 15,
"up_primary": 113
},
{
"first": 5987,
"last": 6004,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
103
],
"primary": 15,
"up_primary": 113
},
{
"first": 6005,
"last": 6015,
"maybe_went_rw": 0,
"up": [
2147483647,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
103
],
"primary": 15,
"up_primary": 66
},
{
"first": 6016,
"last": 6047,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
103
],
"primary": 15,
"up_primary": 113
},
{
"first": 6048,
"last": 6066,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
2147483647
],
"acting": [
2147483647,
2147483647,
15,
73,
2147483647
],
"primary": 15,
"up_primary": 113
},
{
"first": 6067,
"last": 6074,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
15,
73,
2147483647
],
"primary": 15,
"up_primary": 113
},
{
"first": 6075,
"last": 6106,
"maybe_went_rw": 0,
"up": [
113,
66,
2147483647,
73,
103
],
"acting": [
2147483647,
2147483647,
2147483647,
73,
2147483647
],
"primary": 73,
"up_primary": 113
},
{
"first": 6107,
"last": 6115,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
2147483647,
2147483647,
2147483647,
73,
2147483647
],
"primary": 73,
"up_primary": 113
},
{
"first": 6116,
"last": 6116,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
2147483647,
103
],
"acting": [
2147483647,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": -1,
"up_primary": 113
},
{
"first": 6117,
"last": 6121,
"maybe_went_rw": 1,
"up": [
113,
66,
15,
2147483647,
103
],
"acting": [
113,
66,
15,
2147483647,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 6122,
"last": 6171,
"maybe_went_rw": 1,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
66,
15,
2147483647,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 6172,
"last": 6173,
"maybe_went_rw": 0,
"up": [
113,
2147483647,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
2147483647,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 6174,
"last": 6206,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
15,
2147483647,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 6207,
"last": 6213,
"maybe_went_rw": 0,
"up": [
113,
66,
2147483647,
73,
103
],
"acting": [
113,
2147483647,
2147483647,
2147483647,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 6214,
"last": 6262,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
103
],
"acting": [
113,
2147483647,
2147483647,
2147483647,
103
],
"primary": 113,
"up_primary": 113
},
{
"first": 6263,
"last": 6265,
"maybe_went_rw": 0,
"up": [
113,
66,
15,
73,
2147483647
],
"acting": [
113,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": 113,
"up_primary": 113
},
{
"first": 6266,
"last": 6266,
"maybe_went_rw": 0,
"up": [
2147483647,
66,
15,
73,
2147483647
],
"acting": [
2147483647,
2147483647,
2147483647,
2147483647,
2147483647
],
"primary": -1,
"up_primary": 66
},
{
"first": 6267,
"last": 6268,
"maybe_went_rw": 0,
"up": [
2147483647,
66,
15,
73,
2147483647
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"primary": 66,
"up_primary": 66
},
{
"first": 6269,
"last": 6269,
"maybe_went_rw": 0,
"up": [
2147483647,
66,
15,
73,
103
],
"acting": [
2147483647,
66,
15,
73,
2147483647
],
"primary": 66,
"up_primary": 66
}
],
"probing_osds": [
"15(2)",
"66(1)",
"73(3)",
"103(4)",
"113(0)"
],
"down_osds_we_would_probe": [
6
],
"peering_blocked_by": [],
"peering_blocked_by_detail": [
{
"detail": "peering_blocked_by_history_les_bound"
}
]
},
{
"name": "Started",
"enter_time": "2017-03-29 13:48:34.635031"
}
],
"agent_state": {}
}
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
