I have 4 PGs stuck inactive.....
I have tried everything I can find online, and could use some help.
I had 3 OSDs... was in process of rebooting one and another totally crashed and corrupted its filesystem (BAD).
So now I have 4 incomplete PGs.
Things I've tried:
Rebooting all OSDs.
Running 'ceph pg {pg-id} mark_unfound_lost revert'
Running 'ceph osd lost {OSD ID}'
[[{ "state": "down+incomplete",
"epoch": 7994,
"up": [
1,
2],
"acting": [
1,
2],
"info": { "pgid": "4.1ca",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 75,
"last_epoch_started": 6719,
"last_epoch_clean": 6710,
"last_epoch_split": 0,
"same_up_since": 7973,
"same_interval_since": 7973,
"same_primary_since": 7969,
"last_scrub": "6239'25314",
"last_scrub_stamp": "2014-01-05 18:48:01.122717",
"last_deep_scrub": "6235'25313",
"last_deep_scrub_stamp": "2014-01-03 18:47:48.390112",
"last_clean_scrub_stamp": "2014-01-05 18:48:01.122717"},
"stats": { "version": "0'0",
"reported_seq": "1259",
"reported_epoch": "7994",
"state": "down+incomplete",
"last_fresh": "2014-01-07 14:04:36.269587",
"last_change": "2014-01-07 14:04:36.269587",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2014-01-07 14:04:36.269587",
"mapping_epoch": 7971,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 75,
"last_epoch_clean": 6710,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "6239'25314",
"last_scrub_stamp": "2014-01-05 18:48:01.122717",
"last_deep_scrub": "6235'25313",
"last_deep_scrub_stamp": "2014-01-03 18:47:48.390112",
"last_clean_scrub_stamp": "2014-01-05 18:48:01.122717",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0},
"stat_cat_sum": {},
"up": [
1,
2],
"acting": [
1,
2]},
"empty": 1,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 0},
"recovery_state": [
{ "name": "Started\/Primary\/Peering",
"enter_time": "2014-01-07 14:04:36.269509",
"past_intervals": [
{ "first": 5221,
"last": 5226,
"maybe_went_rw": 1,
"up": [
0,
1],
"acting": [
0,
1]},
{ "first": 5227,
"last": 5439,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 5440,
"last": 5441,
"maybe_went_rw": 0,
"up": [
1],
"acting": [
1]},
{ "first": 5442,
"last": 6388,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 6389,
"last": 6395,
"maybe_went_rw": 1,
"up": [
0],
"acting": [
0]},
{ "first": 6396,
"last": 6708,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 6709,
"last": 6717,
"maybe_went_rw": 1,
"up": [
1],
"acting": [
1]},
{ "first": 6718,
"last": 6721,
"maybe_went_rw": 1,
"up": [
1,
2],
"acting": [
1,
2]},
{ "first": 6722,
"last": 6724,
"maybe_went_rw": 1,
"up": [
2],
"acting": [
2]},
{ "first": 6725,
"last": 6737,
"maybe_went_rw": 1,
"up": [
0,
2],
"acting": [
0,
2]},
{ "first": 6738,
"last": 7946,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 7947,
"last": 7948,
"maybe_went_rw": 1,
"up": [
1],
"acting": [
1]},
{ "first": 7949,
"last": 7950,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 7951,
"last": 7952,
"maybe_went_rw": 1,
"up": [
0],
"acting": [
0]},
{ "first": 7953,
"last": 7954,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 7955,
"last": 7956,
"maybe_went_rw": 1,
"up": [
1],
"acting": [
1]},
{ "first": 7957,
"last": 7958,
"maybe_went_rw": 1,
"up": [
1,
2],
"acting": [
1,
2]},
{ "first": 7959,
"last": 7966,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 7967,
"last": 7968,
"maybe_went_rw": 1,
"up": [
0],
"acting": [
0]},
{ "first": 7969,
"last": 7970,
"maybe_went_rw": 1,
"up": [
1,
0],
"acting": [
1,
0]},
{ "first": 7971,
"last": 7972,
"maybe_went_rw": 1,
"up": [
1],
"acting": [
1]}],
"probing_osds": [
1,
2],
"down_osds_we_would_probe": [
0],
"peering_blocked_by": []},
{ "name": "Started",
"enter_time": "2014-01-07 14:04:36.269462"}]}
CEPH OSD TREE
ceph osd tree
# id weight type name up/down reweight
-1 4.5 pool default
-4 1.8 host ceph2
1 1.8 osd.1 up 1
-6 2.7 host ceph3
2 2.7 osd.2 up 1
# id weight type name up/down reweight
-1 4.5 pool default
-4 1.8 host ceph2
1 1.8 osd.1 up 1
-6 2.7 host ceph3
2 2.7 osd.2 up 1
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
