Hello,
Today we had a power failure in a rack housing our OSD servers. We had
7 of our 30 total OSD nodes down. Of the affect PG 2 out of the 3 OSDs
went down.
After everything was back and mostly healthy I found one placement
group marked as incomplete. I can't figure out why.
I'm running ceph 0.94.6 on CentOS7. The following steps have been tried
in this order:
1) Reduce the min_size from 2 to 1 (as suggested by ceph health detail)
2) Set the 2 OSDs that were down to 'out' (one by one) and waited for
the cluster to recover. (this did not work, I set them back in)
3) use ceph-objectstore-tool to export the pg from the 2 osds that went
down, then removed it, restarted the osds.
4) When this did not work, import the data exported from the unaffected
OSD into the two remaining osds.
5) Import the data from the unaffected OSD into all osds that are noted
in "probing_osds"
None of these had any effect on the stuck incomplete PG. I have
attached the output of "ceph pg 54.3e9 query", "ceph health detail", as
well as "ceph -s"
The pool in question is largely read-only (it is an openstack rbd image
pool) so I can leave it like this for the time being. Help would be
very much appreciated!
Thank you,
- Hein-Pieter van Braam
# ceph pg 54.3e9 query
{
"state": "incomplete",
"snap_trimq": "[]",
"epoch": 90440,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"info": {
"pgid": "54.3e9",
"last_update": "88883'4245",
"last_complete": "88883'4245",
"log_tail": "9236'1245",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~b,d~2b,39~27,61~1,64~f,74~1,77~1,7f~b,8b~26,b3~c,c1~19,db~1,dd~a,e8~10,f9~1,ff~12,112~af,1c2~c3,286~1c,2b8~59,312~2,315~6,31c~2c,349~2,34c~6,353~2,356~14,36c~1,36f~f,37f~2,382~5,388~1,38a~5,390~2,393~2,396~2,399~2,39c~2,39f~2,3a2~4,3a7~2,3aa~2,3ad~2,3b0~2,3b3~2,3b6~2,3b9~2,3bc~2,3bf~2,3c2~2,3c5~2,3c8~2,3cb~2,3ce~2,3d1~2,3d4~2,3d7~2,3da~2,3dd~2,3e0~2,3e3~2,3e6~2,3e9~2,3ec~2,3ef~2,3f2~3,3f6~2,3f9~2,3fc~2,3ff~2,402~2,405~2,408~2,40b~2,40e~2,411~2,414~2,417~2,41a~2,41d~2,420~2,423~2,426~2,429~2,42c~2,42f~2,432~2,435~2,438~2,43b~2,43e~2,441~2,444~2,447~2,44a~2,44d~2,450~2,453~2,456~2,459~2,45c~2,45f~2,462~2,465~2,468~2,46b~2,46e~2,471~1,473~1,475~2,478~2,47b~2,47e~2,481~2,484~2,487~2,48a~1,48d~1,48f~2,492~2,495~2,498~2,49b~2,49e~2,4a1~2,4a4~2,4a7~2,4aa~2,4ad~2,4b0~2,4b3~2,4b6~2,4b9~2,4bc~2,4bf~2,4c2~2,4c5~2,4c8~2,4cb~2,4ce~2,4d1~2,4d4~2,4d7~2,4da~2,4dd~2,4e0~2,4e3~2,4e6~2,4e9~2,4ec~2,4ef~2,4f2~2,4f5~2,4f8~2,4fb~2,4fe~6,505~2,508~1,50a~1,50c~2,50f~2,512~2,515~2,518~2,51b~2,51e~2,521~1,523~1,525~2,528~2,52b~2,52e~1]",
"history": {
"epoch_created": 687,
"last_epoch_started": 89166,
"last_epoch_clean": 89793,
"last_epoch_split": 0,
"same_up_since": 90435,
"same_interval_since": 90439,
"same_primary_since": 90435,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007"
},
"stats": {
"version": "88883'4245",
"reported_seq": "27180",
"reported_epoch": "90439",
"state": "incomplete",
"last_fresh": "2016-05-17 13:48:50.546170",
"last_change": "2016-05-17 13:48:50.546170",
"last_active": "2016-05-17 02:48:50.136157",
"last_peered": "2016-05-17 07:44:05.962663",
"last_clean": "2016-05-17 02:48:50.136157",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-17 13:48:50.546170",
"last_undegraded": "2016-05-17 13:48:50.546170",
"last_fullsized": "2016-05-17 13:48:50.546170",
"mapping_epoch": 90433,
"log_start": "9236'1245",
"ondisk_log_start": "9236'1245",
"created": 687,
"last_epoch_clean": 89793,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1140851246,
"num_objects": 770,
"num_object_clones": 0,
"num_object_copies": 2310,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 763,
"num_whiteouts": 627,
"num_read": 10229,
"num_read_kb": 7946011,
"num_write": 2167,
"num_write_kb": 6225921,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8869,
"num_bytes_recovered": 16668174937,
"num_keys_recovered": 0,
"num_objects_omap": 2,
"num_objects_hit_set_archive": 6,
"num_bytes_hit_set_archive": 539
},
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"blocked_by": [],
"up_primary": 32,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 88796,
"hit_set_history": {
"current_last_update": "88863'4231",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": [
{
"begin": "2016-05-14 02:48:48.243213",
"end": "2016-05-14 12:48:08.256405",
"version": "88834'4183"
},
{
"begin": "2016-05-14 12:48:08.256751",
"end": "2016-05-15 02:37:54.958748",
"version": "88834'4185"
},
{
"begin": "2016-05-15 02:48:58.540051",
"end": "2016-05-15 21:46:34.196893",
"version": "88859'4196"
},
{
"begin": "2016-05-15 21:46:34.197299",
"end": "2016-05-15 21:57:39.868962",
"version": "88859'4212"
},
{
"begin": "2016-05-15 22:01:44.215406",
"end": "2016-05-16 02:36:00.358607",
"version": "88860'4224"
},
{
"begin": "2016-05-16 02:48:26.449070",
"end": "2016-05-17 02:36:42.337507",
"version": "88877'4232"
}
]
}
},
"peer_info": [
{
"peer": "4",
"pgid": "54.3e9",
"last_update": "88883'4245",
"last_complete": "88883'4245",
"log_tail": "9236'1245",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~b,d~2b,39~27,61~1,64~f,74~1,77~1,7f~b,8b~26,b3~c,c1~19,db~1,dd~a,e8~10,f9~1,ff~12,112~af,1c2~c3,286~1c,2b8~59,312~2,315~6,31c~2c,349~2,34c~6,353~2,356~14,36c~1,36f~f,37f~2,382~5,388~1,38a~5,390~2,393~2,396~2,399~2,39c~2,39f~2,3a2~4,3a7~2,3aa~2,3ad~2,3b0~2,3b3~2,3b6~2,3b9~2,3bc~2,3bf~2,3c2~2,3c5~2,3c8~2,3cb~2,3ce~2,3d1~2,3d4~2,3d7~2,3da~2,3dd~2,3e0~2,3e3~2,3e6~2,3e9~2,3ec~2,3ef~2,3f2~3,3f6~2,3f9~2,3fc~2,3ff~2,402~2,405~2,408~2,40b~2,40e~2,411~2,414~2,417~2,41a~2,41d~2,420~2,423~2,426~2,429~2,42c~2,42f~2,432~2,435~2,438~2,43b~2,43e~2,441~2,444~2,447~2,44a~2,44d~2,450~2,453~2,456~2,459~2,45c~2,45f~2,462~2,465~2,468~2,46b~2,46e~2,471~1,473~1,475~2,478~2,47b~2,47e~2,481~2,484~2,487~2,48a~1,48d~1,48f~2,492~2,495~2,498~2,49b~2,49e~2,4a1~2,4a4~2,4a7~2,4aa~2,4ad~2,4b0~2,4b3~2,4b6~2,4b9~2,4bc~2,4bf~2,4c2~2,4c5~2,4c8~2,4cb~2,4ce~2,4d1~2,4d4~2,4d7~2,4da~2,4dd~2,4e0~2,4e3~2,4e6~2,4e9~2,4ec~2,4ef~2,4f2~2,4f5~2,4f8~2,4fb~2,4fe~6,505~2,508~1,50a~1,50c~2,50f~2,512~2,515~2,518~2,51b~2,51e~2,521~1,523~1,525~2,528~2,52b~2,52e~1]",
"history": {
"epoch_created": 687,
"last_epoch_started": 89166,
"last_epoch_clean": 89793,
"last_epoch_split": 0,
"same_up_since": 90435,
"same_interval_since": 90439,
"same_primary_since": 90435,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007"
},
"stats": {
"version": "88883'4245",
"reported_seq": "27165",
"reported_epoch": "89793",
"state": "undersized+degraded+remapped+backfilling+peered",
"last_fresh": "2016-05-17 07:44:05.962663",
"last_change": "2016-05-17 07:43:51.436170",
"last_active": "2016-05-17 02:48:50.136157",
"last_peered": "2016-05-17 07:44:05.962663",
"last_clean": "2016-05-17 02:48:50.136157",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-17 07:44:05.962663",
"last_undegraded": "2016-05-17 07:43:50.895774",
"last_fullsized": "2016-05-17 07:43:50.895774",
"mapping_epoch": 90433,
"log_start": "9231'1238",
"ondisk_log_start": "9231'1238",
"created": 687,
"last_epoch_clean": 89166,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007",
"log_size": 3007,
"ondisk_log_size": 3007,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1140851246,
"num_objects": 770,
"num_object_clones": 0,
"num_object_copies": 2310,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 1540,
"num_objects_misplaced": 2,
"num_objects_unfound": 0,
"num_objects_dirty": 763,
"num_whiteouts": 627,
"num_read": 10229,
"num_read_kb": 7946011,
"num_write": 2167,
"num_write_kb": 6225921,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8869,
"num_bytes_recovered": 16668174937,
"num_keys_recovered": 0,
"num_objects_omap": 2,
"num_objects_hit_set_archive": 6,
"num_bytes_hit_set_archive": 539
},
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"blocked_by": [],
"up_primary": 32,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 88796,
"hit_set_history": {
"current_last_update": "88863'4231",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": [
{
"begin": "2016-05-14 02:48:48.243213",
"end": "2016-05-14 12:48:08.256405",
"version": "88834'4183"
},
{
"begin": "2016-05-14 12:48:08.256751",
"end": "2016-05-15 02:37:54.958748",
"version": "88834'4185"
},
{
"begin": "2016-05-15 02:48:58.540051",
"end": "2016-05-15 21:46:34.196893",
"version": "88859'4196"
},
{
"begin": "2016-05-15 21:46:34.197299",
"end": "2016-05-15 21:57:39.868962",
"version": "88859'4212"
},
{
"begin": "2016-05-15 22:01:44.215406",
"end": "2016-05-16 02:36:00.358607",
"version": "88860'4224"
},
{
"begin": "2016-05-16 02:48:26.449070",
"end": "2016-05-17 02:36:42.337507",
"version": "88877'4232"
}
]
}
},
{
"peer": "86",
"pgid": "54.3e9",
"last_update": "88883'4245",
"last_complete": "88883'4245",
"log_tail": "9236'1245",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~b,d~2b,39~27,61~1,64~f,74~1,77~1,7f~b,8b~26,b3~c,c1~19,db~1,dd~a,e8~10,f9~1,ff~12,112~af,1c2~c3,286~1c,2b8~59,312~2,315~6,31c~2c,349~2,34c~6,353~2,356~14,36c~1,36f~f,37f~2,382~5,388~1,38a~5,390~2,393~2,396~2,399~2,39c~2,39f~2,3a2~4,3a7~2,3aa~2,3ad~2,3b0~2,3b3~2,3b6~2,3b9~2,3bc~2,3bf~2,3c2~2,3c5~2,3c8~2,3cb~2,3ce~2,3d1~2,3d4~2,3d7~2,3da~2,3dd~2,3e0~2,3e3~2,3e6~2,3e9~2,3ec~2,3ef~2,3f2~3,3f6~2,3f9~2,3fc~2,3ff~2,402~2,405~2,408~2,40b~2,40e~2,411~2,414~2,417~2,41a~2,41d~2,420~2,423~2,426~2,429~2,42c~2,42f~2,432~2,435~2,438~2,43b~2,43e~2,441~2,444~2,447~2,44a~2,44d~2,450~2,453~2,456~2,459~2,45c~2,45f~2,462~2,465~2,468~2,46b~2,46e~2,471~1,473~1,475~2,478~2,47b~2,47e~2,481~2,484~2,487~2,48a~1,48d~1,48f~2,492~2,495~2,498~2,49b~2,49e~2,4a1~2,4a4~2,4a7~2,4aa~2,4ad~2,4b0~2,4b3~2,4b6~2,4b9~2,4bc~2,4bf~2,4c2~2,4c5~2,4c8~2,4cb~2,4ce~2,4d1~2,4d4~2,4d7~2,4da~2,4dd~2,4e0~2,4e3~2,4e6~2,4e9~2,4ec~2,4ef~2,4f2~2,4f5~2,4f8~2,4fb~2,4fe~6,505~2,508~1,50a~1,50c~2,50f~2,512~2,515~2,518~2,51b~2,51e~2,521~1,523~1,525~2,528~2,52b~2,52e~1]",
"history": {
"epoch_created": 687,
"last_epoch_started": 89166,
"last_epoch_clean": 89793,
"last_epoch_split": 0,
"same_up_since": 90435,
"same_interval_since": 90439,
"same_primary_since": 90435,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007"
},
"stats": {
"version": "88883'4245",
"reported_seq": "27165",
"reported_epoch": "89793",
"state": "undersized+degraded+remapped+backfilling+peered",
"last_fresh": "2016-05-17 07:44:05.962663",
"last_change": "2016-05-17 07:43:51.436170",
"last_active": "2016-05-17 02:48:50.136157",
"last_peered": "2016-05-17 07:44:05.962663",
"last_clean": "2016-05-17 02:48:50.136157",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-17 07:44:05.962663",
"last_undegraded": "2016-05-17 07:43:50.895774",
"last_fullsized": "2016-05-17 07:43:50.895774",
"mapping_epoch": 90433,
"log_start": "9231'1238",
"ondisk_log_start": "9231'1238",
"created": 687,
"last_epoch_clean": 89166,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007",
"log_size": 3007,
"ondisk_log_size": 3007,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1140851246,
"num_objects": 770,
"num_object_clones": 0,
"num_object_copies": 2310,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 1540,
"num_objects_misplaced": 2,
"num_objects_unfound": 0,
"num_objects_dirty": 763,
"num_whiteouts": 627,
"num_read": 10229,
"num_read_kb": 7946011,
"num_write": 2167,
"num_write_kb": 6225921,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8869,
"num_bytes_recovered": 16668174937,
"num_keys_recovered": 0,
"num_objects_omap": 2,
"num_objects_hit_set_archive": 6,
"num_bytes_hit_set_archive": 539
},
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"blocked_by": [],
"up_primary": 32,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 88796,
"hit_set_history": {
"current_last_update": "88863'4231",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": [
{
"begin": "2016-05-14 02:48:48.243213",
"end": "2016-05-14 12:48:08.256405",
"version": "88834'4183"
},
{
"begin": "2016-05-14 12:48:08.256751",
"end": "2016-05-15 02:37:54.958748",
"version": "88834'4185"
},
{
"begin": "2016-05-15 02:48:58.540051",
"end": "2016-05-15 21:46:34.196893",
"version": "88859'4196"
},
{
"begin": "2016-05-15 21:46:34.197299",
"end": "2016-05-15 21:57:39.868962",
"version": "88859'4212"
},
{
"begin": "2016-05-15 22:01:44.215406",
"end": "2016-05-16 02:36:00.358607",
"version": "88860'4224"
},
{
"begin": "2016-05-16 02:48:26.449070",
"end": "2016-05-17 02:36:42.337507",
"version": "88877'4232"
}
]
}
},
{
"peer": "96",
"pgid": "54.3e9",
"last_update": "88883'4245",
"last_complete": "88883'4245",
"log_tail": "9236'1245",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~b,d~2b,39~27,61~1,64~f,74~1,77~1,7f~b,8b~26,b3~c,c1~19,db~1,dd~a,e8~10,f9~1,ff~12,112~af,1c2~c3,286~1c,2b8~59,312~2,315~6,31c~2c,349~2,34c~6,353~2,356~14,36c~1,36f~f,37f~2,382~5,388~1,38a~5,390~2,393~2,396~2,399~2,39c~2,39f~2,3a2~4,3a7~2,3aa~2,3ad~2,3b0~2,3b3~2,3b6~2,3b9~2,3bc~2,3bf~2,3c2~2,3c5~2,3c8~2,3cb~2,3ce~2,3d1~2,3d4~2,3d7~2,3da~2,3dd~2,3e0~2,3e3~2,3e6~2,3e9~2,3ec~2,3ef~2,3f2~3,3f6~2,3f9~2,3fc~2,3ff~2,402~2,405~2,408~2,40b~2,40e~2,411~2,414~2,417~2,41a~2,41d~2,420~2,423~2,426~2,429~2,42c~2,42f~2,432~2,435~2,438~2,43b~2,43e~2,441~2,444~2,447~2,44a~2,44d~2,450~2,453~2,456~2,459~2,45c~2,45f~2,462~2,465~2,468~2,46b~2,46e~2,471~1,473~1,475~2,478~2,47b~2,47e~2,481~2,484~2,487~2,48a~1,48d~1,48f~2,492~2,495~2,498~2,49b~2,49e~2,4a1~2,4a4~2,4a7~2,4aa~2,4ad~2,4b0~2,4b3~2,4b6~2,4b9~2,4bc~2,4bf~2,4c2~2,4c5~2,4c8~2,4cb~2,4ce~2,4d1~2,4d4~2,4d7~2,4da~2,4dd~2,4e0~2,4e3~2,4e6~2,4e9~2,4ec~2,4ef~2,4f2~2,4f5~2,4f8~2,4fb~2,4fe~6,505~2,508~1,50a~1,50c~2,50f~2,512~2,515~2,518~2,51b~2,51e~2,521~1,523~1,525~2,528~2,52b~2,52e~1]",
"history": {
"epoch_created": 687,
"last_epoch_started": 89166,
"last_epoch_clean": 89793,
"last_epoch_split": 0,
"same_up_since": 90435,
"same_interval_since": 90439,
"same_primary_since": 90435,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007"
},
"stats": {
"version": "88883'4245",
"reported_seq": "27165",
"reported_epoch": "89793",
"state": "undersized+degraded+remapped+backfilling+peered",
"last_fresh": "2016-05-17 07:44:05.962663",
"last_change": "2016-05-17 07:43:51.436170",
"last_active": "2016-05-17 02:48:50.136157",
"last_peered": "2016-05-17 07:44:05.962663",
"last_clean": "2016-05-17 02:48:50.136157",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-17 07:44:05.962663",
"last_undegraded": "2016-05-17 07:43:50.895774",
"last_fullsized": "2016-05-17 07:43:50.895774",
"mapping_epoch": 90433,
"log_start": "9231'1238",
"ondisk_log_start": "9231'1238",
"created": 687,
"last_epoch_clean": 89166,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007",
"log_size": 3007,
"ondisk_log_size": 3007,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1140851246,
"num_objects": 770,
"num_object_clones": 0,
"num_object_copies": 2310,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 1540,
"num_objects_misplaced": 2,
"num_objects_unfound": 0,
"num_objects_dirty": 763,
"num_whiteouts": 627,
"num_read": 10229,
"num_read_kb": 7946011,
"num_write": 2167,
"num_write_kb": 6225921,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8869,
"num_bytes_recovered": 16668174937,
"num_keys_recovered": 0,
"num_objects_omap": 2,
"num_objects_hit_set_archive": 6,
"num_bytes_hit_set_archive": 539
},
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"blocked_by": [],
"up_primary": 32,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 88796,
"hit_set_history": {
"current_last_update": "88863'4231",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": [
{
"begin": "2016-05-14 02:48:48.243213",
"end": "2016-05-14 12:48:08.256405",
"version": "88834'4183"
},
{
"begin": "2016-05-14 12:48:08.256751",
"end": "2016-05-15 02:37:54.958748",
"version": "88834'4185"
},
{
"begin": "2016-05-15 02:48:58.540051",
"end": "2016-05-15 21:46:34.196893",
"version": "88859'4196"
},
{
"begin": "2016-05-15 21:46:34.197299",
"end": "2016-05-15 21:57:39.868962",
"version": "88859'4212"
},
{
"begin": "2016-05-15 22:01:44.215406",
"end": "2016-05-16 02:36:00.358607",
"version": "88860'4224"
},
{
"begin": "2016-05-16 02:48:26.449070",
"end": "2016-05-17 02:36:42.337507",
"version": "88877'4232"
}
]
}
},
{
"peer": "161",
"pgid": "54.3e9",
"last_update": "88883'4245",
"last_complete": "88883'4245",
"log_tail": "9236'1245",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~b,d~2b,39~27,61~1,64~f,74~1,77~1,7f~b,8b~26,b3~c,c1~19,db~1,dd~a,e8~10,f9~1,ff~12,112~af,1c2~c3,286~1c,2b8~59,312~2,315~6,31c~2c,349~2,34c~6,353~2,356~14,36c~1,36f~f,37f~2,382~5,388~1,38a~5,390~2,393~2,396~2,399~2,39c~2,39f~2,3a2~4,3a7~2,3aa~2,3ad~2,3b0~2,3b3~2,3b6~2,3b9~2,3bc~2,3bf~2,3c2~2,3c5~2,3c8~2,3cb~2,3ce~2,3d1~2,3d4~2,3d7~2,3da~2,3dd~2,3e0~2,3e3~2,3e6~2,3e9~2,3ec~2,3ef~2,3f2~3,3f6~2,3f9~2,3fc~2,3ff~2,402~2,405~2,408~2,40b~2,40e~2,411~2,414~2,417~2,41a~2,41d~2,420~2,423~2,426~2,429~2,42c~2,42f~2,432~2,435~2,438~2,43b~2,43e~2,441~2,444~2,447~2,44a~2,44d~2,450~2,453~2,456~2,459~2,45c~2,45f~2,462~2,465~2,468~2,46b~2,46e~2,471~1,473~1,475~2,478~2,47b~2,47e~2,481~2,484~2,487~2,48a~1,48d~1,48f~2,492~2,495~2,498~2,49b~2,49e~2,4a1~2,4a4~2,4a7~2,4aa~2,4ad~2,4b0~2,4b3~2,4b6~2,4b9~2,4bc~2,4bf~2,4c2~2,4c5~2,4c8~2,4cb~2,4ce~2,4d1~2,4d4~2,4d7~2,4da~2,4dd~2,4e0~2,4e3~2,4e6~2,4e9~2,4ec~2,4ef~2,4f2~2,4f5~2,4f8~2,4fb~2,4fe~6,505~2,508~1,50a~1,50c~2,50f~2,512~2,515~2,518~2,51b~2,51e~2,521~1,523~1,525~2,528~2,52b~2,52e~1]",
"history": {
"epoch_created": 687,
"last_epoch_started": 89166,
"last_epoch_clean": 89793,
"last_epoch_split": 0,
"same_up_since": 90435,
"same_interval_since": 90439,
"same_primary_since": 90435,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007"
},
"stats": {
"version": "88883'4245",
"reported_seq": "27165",
"reported_epoch": "89793",
"state": "undersized+degraded+remapped+backfilling+peered",
"last_fresh": "2016-05-17 07:44:05.962663",
"last_change": "2016-05-17 07:43:51.436170",
"last_active": "2016-05-17 02:48:50.136157",
"last_peered": "2016-05-17 07:44:05.962663",
"last_clean": "2016-05-17 02:48:50.136157",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-17 07:44:05.962663",
"last_undegraded": "2016-05-17 07:43:50.895774",
"last_fullsized": "2016-05-17 07:43:50.895774",
"mapping_epoch": 90433,
"log_start": "9231'1238",
"ondisk_log_start": "9231'1238",
"created": 687,
"last_epoch_clean": 89166,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007",
"log_size": 3007,
"ondisk_log_size": 3007,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1140851246,
"num_objects": 770,
"num_object_clones": 0,
"num_object_copies": 2310,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 1540,
"num_objects_misplaced": 2,
"num_objects_unfound": 0,
"num_objects_dirty": 763,
"num_whiteouts": 627,
"num_read": 10229,
"num_read_kb": 7946011,
"num_write": 2167,
"num_write_kb": 6225921,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8869,
"num_bytes_recovered": 16668174937,
"num_keys_recovered": 0,
"num_objects_omap": 2,
"num_objects_hit_set_archive": 6,
"num_bytes_hit_set_archive": 539
},
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"blocked_by": [],
"up_primary": 32,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 88796,
"hit_set_history": {
"current_last_update": "88863'4231",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": [
{
"begin": "2016-05-14 02:48:48.243213",
"end": "2016-05-14 12:48:08.256405",
"version": "88834'4183"
},
{
"begin": "2016-05-14 12:48:08.256751",
"end": "2016-05-15 02:37:54.958748",
"version": "88834'4185"
},
{
"begin": "2016-05-15 02:48:58.540051",
"end": "2016-05-15 21:46:34.196893",
"version": "88859'4196"
},
{
"begin": "2016-05-15 21:46:34.197299",
"end": "2016-05-15 21:57:39.868962",
"version": "88859'4212"
},
{
"begin": "2016-05-15 22:01:44.215406",
"end": "2016-05-16 02:36:00.358607",
"version": "88860'4224"
},
{
"begin": "2016-05-16 02:48:26.449070",
"end": "2016-05-17 02:36:42.337507",
"version": "88877'4232"
}
]
}
},
{
"peer": "166",
"pgid": "54.3e9",
"last_update": "88883'4245",
"last_complete": "88883'4245",
"log_tail": "9236'1245",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[1~b,d~2b,39~27,61~1,64~f,74~1,77~1,7f~b,8b~26,b3~c,c1~19,db~1,dd~a,e8~10,f9~1,ff~12,112~af,1c2~c3,286~1c,2b8~59,312~2,315~6,31c~2c,349~2,34c~6,353~2,356~14,36c~1,36f~f,37f~2,382~5,388~1,38a~5,390~2,393~2,396~2,399~2,39c~2,39f~2,3a2~4,3a7~2,3aa~2,3ad~2,3b0~2,3b3~2,3b6~2,3b9~2,3bc~2,3bf~2,3c2~2,3c5~2,3c8~2,3cb~2,3ce~2,3d1~2,3d4~2,3d7~2,3da~2,3dd~2,3e0~2,3e3~2,3e6~2,3e9~2,3ec~2,3ef~2,3f2~3,3f6~2,3f9~2,3fc~2,3ff~2,402~2,405~2,408~2,40b~2,40e~2,411~2,414~2,417~2,41a~2,41d~2,420~2,423~2,426~2,429~2,42c~2,42f~2,432~2,435~2,438~2,43b~2,43e~2,441~2,444~2,447~2,44a~2,44d~2,450~2,453~2,456~2,459~2,45c~2,45f~2,462~2,465~2,468~2,46b~2,46e~2,471~1,473~1,475~2,478~2,47b~2,47e~2,481~2,484~2,487~2,48a~1,48d~1,48f~2,492~2,495~2,498~2,49b~2,49e~2,4a1~2,4a4~2,4a7~2,4aa~2,4ad~2,4b0~2,4b3~2,4b6~2,4b9~2,4bc~2,4bf~2,4c2~2,4c5~2,4c8~2,4cb~2,4ce~2,4d1~2,4d4~2,4d7~2,4da~2,4dd~2,4e0~2,4e3~2,4e6~2,4e9~2,4ec~2,4ef~2,4f2~2,4f5~2,4f8~2,4fb~2,4fe~6,505~2,508~1,50a~1,50c~2,50f~2,512~2,515~2,518~2,51b~2,51e~2,521~1,523~1,525~2,528~2,52b~2,52e~1]",
"history": {
"epoch_created": 687,
"last_epoch_started": 89166,
"last_epoch_clean": 89793,
"last_epoch_split": 0,
"same_up_since": 90435,
"same_interval_since": 90439,
"same_primary_since": 90435,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007"
},
"stats": {
"version": "88883'4245",
"reported_seq": "27173",
"reported_epoch": "90433",
"state": "incomplete",
"last_fresh": "2016-05-17 13:35:18.859350",
"last_change": "2016-05-17 13:35:18.859350",
"last_active": "2016-05-17 02:48:50.136157",
"last_peered": "2016-05-17 07:44:05.962663",
"last_clean": "2016-05-17 02:48:50.136157",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "2016-05-17 13:35:18.859350",
"last_undegraded": "2016-05-17 13:35:18.859350",
"last_fullsized": "2016-05-17 13:35:18.859350",
"mapping_epoch": 90433,
"log_start": "9236'1245",
"ondisk_log_start": "9236'1245",
"created": 687,
"last_epoch_clean": 89793,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "88562'4134",
"last_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_deep_scrub": "88562'4134",
"last_deep_scrub_stamp": "2016-05-11 17:52:57.157007",
"last_clean_scrub_stamp": "2016-05-11 17:52:57.157007",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": {
"num_bytes": 1140851246,
"num_objects": 770,
"num_object_clones": 0,
"num_object_copies": 2310,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 763,
"num_whiteouts": 627,
"num_read": 10229,
"num_read_kb": 7946011,
"num_write": 2167,
"num_write_kb": 6225921,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 8869,
"num_bytes_recovered": 16668174937,
"num_keys_recovered": 0,
"num_objects_omap": 2,
"num_objects_hit_set_archive": 6,
"num_bytes_hit_set_archive": 539
},
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"blocked_by": [
32
],
"up_primary": 32,
"acting_primary": 32
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 88796,
"hit_set_history": {
"current_last_update": "88863'4231",
"current_last_stamp": "0.000000",
"current_info": {
"begin": "0.000000",
"end": "0.000000",
"version": "0'0"
},
"history": [
{
"begin": "2016-05-14 02:48:48.243213",
"end": "2016-05-14 12:48:08.256405",
"version": "88834'4183"
},
{
"begin": "2016-05-14 12:48:08.256751",
"end": "2016-05-15 02:37:54.958748",
"version": "88834'4185"
},
{
"begin": "2016-05-15 02:48:58.540051",
"end": "2016-05-15 21:46:34.196893",
"version": "88859'4196"
},
{
"begin": "2016-05-15 21:46:34.197299",
"end": "2016-05-15 21:57:39.868962",
"version": "88859'4212"
},
{
"begin": "2016-05-15 22:01:44.215406",
"end": "2016-05-16 02:36:00.358607",
"version": "88860'4224"
},
{
"begin": "2016-05-16 02:48:26.449070",
"end": "2016-05-17 02:36:42.337507",
"version": "88877'4232"
}
]
}
}
],
"recovery_state": [
{
"name": "Started\/Primary\/Peering",
"enter_time": "2016-05-17 13:48:50.536517",
"past_intervals": [
{
"first": 89793,
"last": 89800,
"maybe_went_rw": 0,
"up": [
32,
166,
96
],
"acting": [
32
],
"primary": 32,
"up_primary": 32
},
{
"first": 89801,
"last": 89859,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89860,
"last": 89861,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89862,
"last": 89865,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89866,
"last": 89876,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89877,
"last": 89880,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 89881,
"last": 89883,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89884,
"last": 89889,
"maybe_went_rw": 1,
"up": [
32,
96
],
"acting": [
32,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89890,
"last": 89891,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89892,
"last": 89893,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 89894,
"last": 89897,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 89898,
"last": 90136,
"maybe_went_rw": 1,
"up": [
161,
86,
4
],
"acting": [
161,
86,
4
],
"primary": 161,
"up_primary": 161
},
{
"first": 90137,
"last": 90320,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90321,
"last": 90326,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 90327,
"last": 90330,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90331,
"last": 90335,
"maybe_went_rw": 1,
"up": [
32,
96
],
"acting": [
32,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90336,
"last": 90339,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90340,
"last": 90342,
"maybe_went_rw": 1,
"up": [
32,
166
],
"acting": [
32,
166
],
"primary": 32,
"up_primary": 32
},
{
"first": 90343,
"last": 90346,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90347,
"last": 90350,
"maybe_went_rw": 1,
"up": [
32,
96
],
"acting": [
32,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90351,
"last": 90354,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90355,
"last": 90359,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 90360,
"last": 90373,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90374,
"last": 90378,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90379,
"last": 90397,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90398,
"last": 90400,
"maybe_went_rw": 1,
"up": [
32,
96
],
"acting": [
32,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90401,
"last": 90404,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90405,
"last": 90406,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 90407,
"last": 90408,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90409,
"last": 90410,
"maybe_went_rw": 1,
"up": [
32,
96
],
"acting": [
32,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90411,
"last": 90412,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90413,
"last": 90414,
"maybe_went_rw": 1,
"up": [
32,
96
],
"acting": [
32,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90428,
"last": 90428,
"maybe_went_rw": 0,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90429,
"last": 90430,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 90431,
"last": 90432,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90433,
"last": 90434,
"maybe_went_rw": 1,
"up": [
166,
96
],
"acting": [
166,
96
],
"primary": 166,
"up_primary": 166
},
{
"first": 90435,
"last": 90436,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
},
{
"first": 90437,
"last": 90438,
"maybe_went_rw": 1,
"up": [
32,
166,
96
],
"acting": [
32,
166,
96
],
"primary": 32,
"up_primary": 32
}
],
"probing_osds": [
"4",
"32",
"86",
"96",
"161",
"166"
],
"down_osds_we_would_probe": [],
"peering_blocked_by": []
},
{
"name": "Started",
"enter_time": "2016-05-17 13:48:50.536469"
}
],
"agent_state": {}
}
# ceph health detail
HEALTH_WARN 1 pgs incomplete; 1 pgs stuck inactive; 1 pgs stuck unclean; 4 requests are blocked > 32 sec; 1 osds have slow requests; noout flag(s) set
pg 54.3e9 is stuck inactive for 43182.455128, current state incomplete, last acting [32,166,96]
pg 54.3e9 is stuck unclean for 43182.455310, current state incomplete, last acting [32,166,96]
pg 54.3e9 is incomplete, acting [32,166,96]
4 ops are blocked > 8388.61 sec
4 ops are blocked > 8388.61 sec on osd.32
1 osds have slow requests
noout flag(s) set
# ceph -s
cluster 42357358-3466-45a4-9eb9-379db0a0eab9
health HEALTH_WARN
1 pgs incomplete
1 pgs stuck inactive
1 pgs stuck unclean
4 requests are blocked > 32 sec
noout flag(s) set
monmap e1: 3 mons at {ams2-openstack-hv-1.domain.com=10.66.0.10:6789/0,ams2-openstack-hv-17.domain.com=10.66.0.26:6789/0,ams2-openstack-hv-2.domain.com=10.66.0.11:6789/0}
election epoch 400, quorum 0,1,2 ams2-openstack-hv-1.domain.com,ams2-openstack-hv-2.domain.com,ams2-openstack-hv-17.domain.com
osdmap e90440: 229 osds: 229 up, 229 in
flags noout
pgmap v13663736: 7680 pgs, 12 pools, 23211 GB data, 9399 kobjects
44249 GB used, 156 TB / 199 TB avail
7679 active+clean
1 incomplete
client io 4403 kB/s rd, 12448 kB/s wr, 2139 op/s
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
