Hi,
I have some problem. After OSD expand, and cluster crush re-organize i
have 1 pg in incomplete state. How can i solve this problem ??
ceph -s
health HEALTH_WARN 1 pgs incomplete; 1 pgs stuck inactive; 1 pgs
stuck unclean
monmap e21: 3 mons at
{0=10.178.64.4:6790/0,1=10.178.64.5:6790/0,2=10.178.64.6:6790/0},
election epoch 54, quorum 0,1,2 0,1,2
osdmap e87682: 156 osds: 156 up, 156 in
pgmap v13097839: 6480 pgs: 6479 active+clean, 1 incomplete; 1484
GB data, 7202 GB used, 36218 GB / 43420 GB avail
mdsmap e1: 0/0/1 up
ceph health details
HEALTH_WARN 1 pgs incomplete; 1 pgs stuck inactive; 1 pgs stuck unclean
pg 5.5c is stuck incomplete, last acting [35,68,120]
pg 5.5c is stuck incomplete, last acting [35,68,120]
pg 5.5c is incomplete, acting [35,68,120]
in attachment output from ceph pg 5.5c query
Regards
Sławek "sZiBis" Skowron
ceph pg 5.5c query
{ "state": "incomplete",
"up": [
35,
68,
120],
"acting": [
35,
68,
120],
"info": { "pgid": "5.5c",
"last_update": "28692'1809",
"last_complete": "28692'1809",
"log_tail": "509'809",
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 365,
"last_epoch_started": 37673,
"last_epoch_clean": 24973,
"last_epoch_split": 37673,
"same_up_since": 81165,
"same_interval_since": 81165,
"same_primary_since": 55011,
"last_scrub": "19046'1806",
"last_scrub_stamp": "2013-02-11 00:20:52.190807"},
"stats": { "version": "28692'1809",
"reported": "55011'57838",
"state": "incomplete",
"last_fresh": "2013-02-20 16:27:35.078140",
"last_change": "2013-02-19 11:14:39.520274",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_unstale": "2013-02-20 16:27:35.078140",
"mapping_epoch": 70925,
"log_start": "509'809",
"ondisk_log_start": "509'809",
"created": 365,
"last_epoch_clean": 365,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "19046'1806",
"last_scrub_stamp": "2013-02-11 00:20:52.190807",
"log_size": 203950,
"ondisk_log_size": 203950,
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0},
"stat_cat_sum": {},
"up": [
35,
68,
120],
"acting": [
35,
68,
120]},
"empty": 0,
"dne": 0,
"incomplete": 1},
"recovery_state": [
{ "name": "Started\/Primary\/Peering",
"enter_time": "2013-02-19 11:14:35.094762",
"past_intervals": [
{ "first": 23374,
"last": 23496,
"maybe_went_rw": 1,
"up": [
95],
"acting": [
95]},
{ "first": 23497,
"last": 23498,
"maybe_went_rw": 1,
"up": [
56,
95],
"acting": [
56,
95]},
{ "first": 23499,
"last": 23540,
"maybe_went_rw": 1,
"up": [
56,
95],
"acting": [
95,
56]},
{ "first": 23541,
"last": 24899,
"maybe_went_rw": 1,
"up": [
56,
95],
"acting": [
56,
95]},
{ "first": 24900,
"last": 24908,
"maybe_went_rw": 1,
"up": [
68,
95],
"acting": [
68,
95]},
{ "first": 24909,
"last": 24950,
"maybe_went_rw": 1,
"up": [
72,
95],
"acting": [
72,
95]},
{ "first": 24951,
"last": 25727,
"maybe_went_rw": 1,
"up": [
72,
95],
"acting": [
72,
95,
56]},
{ "first": 25728,
"last": 25739,
"maybe_went_rw": 1,
"up": [
72,
20,
93],
"acting": [
72,
95,
56]},
{ "first": 25740,
"last": 25760,
"maybe_went_rw": 1,
"up": [
72,
20,
93],
"acting": [
72,
20,
56,
66]},
{ "first": 25761,
"last": 25765,
"maybe_went_rw": 1,
"up": [
72,
24,
93],
"acting": [
72,
20,
56,
66]},
{ "first": 25766,
"last": 26236,
"maybe_went_rw": 1,
"up": [
72,
24,
93],
"acting": [
72,
24,
56,
66]},
{ "first": 26237,
"last": 26237,
"maybe_went_rw": 0,
"up": [
72,
93],
"acting": [
72,
56,
66]},
{ "first": 26238,
"last": 26329,
"maybe_went_rw": 1,
"up": [
72,
24,
93],
"acting": [
72,
24,
56,
66]},
{ "first": 26330,
"last": 26332,
"maybe_went_rw": 1,
"up": [
72,
4,
93],
"acting": [
72,
24,
56,
66]},
{ "first": 26333,
"last": 27674,
"maybe_went_rw": 1,
"up": [
72,
4,
93],
"acting": [
72,
4,
56,
66]},
{ "first": 27675,
"last": 27675,
"maybe_went_rw": 0,
"up": [
72,
93],
"acting": [
72,
56,
66]},
{ "first": 27676,
"last": 30530,
"maybe_went_rw": 1,
"up": [
72,
4,
93],
"acting": [
72,
4,
56,
66]},
{ "first": 30531,
"last": 30536,
"maybe_went_rw": 0,
"up": [
75,
4,
93],
"acting": [
72,
4,
56,
66]},
{ "first": 30537,
"last": 30543,
"maybe_went_rw": 0,
"up": [
77,
4,
93],
"acting": [
72,
4,
56,
66]},
{ "first": 30544,
"last": 30557,
"maybe_went_rw": 1,
"up": [
4,
95,
26],
"acting": [
72,
4,
56,
66]},
{ "first": 30558,
"last": 30590,
"maybe_went_rw": 1,
"up": [
4,
95,
26],
"acting": [
72,
4,
95,
56]},
{ "first": 30591,
"last": 30613,
"maybe_went_rw": 1,
"up": [
4,
95,
26],
"acting": [
72,
4,
95]},
{ "first": 30614,
"last": 30615,
"maybe_went_rw": 1,
"up": [
4,
95,
26],
"acting": [
4,
95]},
{ "first": 30616,
"last": 30643,
"maybe_went_rw": 1,
"up": [
4,
95,
26],
"acting": [
95,
4]},
{ "first": 30644,
"last": 30644,
"maybe_went_rw": 0,
"up": [
4,
46,
93],
"acting": [
95,
4]},
{ "first": 30645,
"last": 30646,
"maybe_went_rw": 1,
"up": [
33,
4,
93],
"acting": [
95,
4]},
{ "first": 30647,
"last": 30648,
"maybe_went_rw": 1,
"up": [
33,
4,
93],
"acting": [
95,
33]},
{ "first": 30649,
"last": 30649,
"maybe_went_rw": 0,
"up": [
35,
4,
93],
"acting": [
95,
33]},
{ "first": 30650,
"last": 30650,
"maybe_went_rw": 0,
"up": [
43,
4,
93],
"acting": [
95,
33]},
{ "first": 30651,
"last": 30653,
"maybe_went_rw": 1,
"up": [
35,
4,
93],
"acting": [
95,
33]},
{ "first": 30654,
"last": 31812,
"maybe_went_rw": 1,
"up": [
35,
4,
93],
"acting": [
95,
35]},
{ "first": 31813,
"last": 31813,
"maybe_went_rw": 0,
"up": [
10,
46,
93],
"acting": [
95,
35]},
{ "first": 31814,
"last": 31814,
"maybe_went_rw": 0,
"up": [
11,
46,
93],
"acting": [
95,
35]},
{ "first": 31815,
"last": 31818,
"maybe_went_rw": 1,
"up": [
35,
4,
93],
"acting": [
95,
35]},
{ "first": 31819,
"last": 31822,
"maybe_went_rw": 1,
"up": [
9,
46,
93],
"acting": [
95,
35]},
{ "first": 31823,
"last": 31823,
"maybe_went_rw": 0,
"up": [
25,
46,
93],
"acting": [
95,
35]},
{ "first": 31824,
"last": 31827,
"maybe_went_rw": 1,
"up": [
12,
46,
93],
"acting": [
95,
35]},
{ "first": 31828,
"last": 31830,
"maybe_went_rw": 1,
"up": [
11,
46,
93],
"acting": [
95,
35]},
{ "first": 31831,
"last": 31831,
"maybe_went_rw": 0,
"up": [
11,
46,
93],
"acting": [
95,
11]},
{ "first": 31832,
"last": 31833,
"maybe_went_rw": 1,
"up": [
35,
4,
93],
"acting": [
95,
11]},
{ "first": 31834,
"last": 31966,
"maybe_went_rw": 1,
"up": [
35,
4,
93],
"acting": [
95,
35]},
{ "first": 31967,
"last": 31974,
"maybe_went_rw": 1,
"up": [
46,
93],
"acting": [
95,
35]},
{ "first": 31975,
"last": 31992,
"maybe_went_rw": 1,
"up": [
46,
93],
"acting": [
95,
46]},
{ "first": 31993,
"last": 31994,
"maybe_went_rw": 1,
"up": [
66,
46,
93],
"acting": [
95,
46]},
{ "first": 31995,
"last": 31995,
"maybe_went_rw": 0,
"up": [
66,
46,
93],
"acting": [
95,
66]},
{ "first": 31996,
"last": 31997,
"maybe_went_rw": 1,
"up": [
46,
93],
"acting": [
95,
66]},
{ "first": 31998,
"last": 32018,
"maybe_went_rw": 1,
"up": [
46,
93],
"acting": [
95,
46]},
{ "first": 32019,
"last": 32048,
"maybe_went_rw": 1,
"up": [
72,
46,
93],
"acting": [
95,
46]},
{ "first": 32049,
"last": 32391,
"maybe_went_rw": 1,
"up": [
72,
46,
93],
"acting": [
95,
72]},
{ "first": 32392,
"last": 37500,
"maybe_went_rw": 1,
"up": [
72,
4,
93],
"acting": [
95,
72]},
{ "first": 37501,
"last": 37501,
"maybe_went_rw": 0,
"up": [
20,
68,
93],
"acting": [
95,
72]},
{ "first": 37502,
"last": 37502,
"maybe_went_rw": 0,
"up": [
10,
68,
93],
"acting": [
95,
72]},
{ "first": 37503,
"last": 37503,
"maybe_went_rw": 0,
"up": [
11,
68,
93],
"acting": [
95,
72]},
{ "first": 37504,
"last": 37505,
"maybe_went_rw": 1,
"up": [
16,
68,
93],
"acting": [
95,
72]},
{ "first": 37506,
"last": 37507,
"maybe_went_rw": 1,
"up": [
16,
68,
93],
"acting": [
95,
16]},
{ "first": 37508,
"last": 37508,
"maybe_went_rw": 0,
"up": [
9,
68,
93],
"acting": [
95,
16]},
{ "first": 37509,
"last": 37509,
"maybe_went_rw": 0,
"up": [
25,
68,
93],
"acting": [
95,
16]},
{ "first": 37510,
"last": 37512,
"maybe_went_rw": 1,
"up": [
12,
68,
93],
"acting": [
95,
16]},
{ "first": 37513,
"last": 37513,
"maybe_went_rw": 0,
"up": [
12,
68,
93],
"acting": [
95,
12]},
{ "first": 37514,
"last": 37514,
"maybe_went_rw": 0,
"up": [
11,
68,
93],
"acting": [
95,
12]},
{ "first": 37515,
"last": 37515,
"maybe_went_rw": 0,
"up": [
7,
68,
93],
"acting": [
95,
12]},
{ "first": 37516,
"last": 37519,
"maybe_went_rw": 1,
"up": [
72,
4,
93],
"acting": [
95,
12]},
{ "first": 37520,
"last": 37624,
"maybe_went_rw": 1,
"up": [
72,
4,
93],
"acting": [
95,
72]},
{ "first": 37625,
"last": 37626,
"maybe_went_rw": 1,
"up": [
68,
93],
"acting": [
95,
72]},
{ "first": 37627,
"last": 37669,
"maybe_went_rw": 1,
"up": [
68,
93],
"acting": [
95,
68]},
{ "first": 37670,
"last": 37671,
"maybe_went_rw": 1,
"up": [
35,
68,
93],
"acting": [
95,
68]},
{ "first": 37672,
"last": 41684,
"maybe_went_rw": 1,
"up": [
35,
68,
93],
"acting": [
95,
35]},
{ "first": 41685,
"last": 41735,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
35]},
{ "first": 41736,
"last": 46921,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
86]},
{ "first": 46922,
"last": 46929,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
95,
86]},
{ "first": 46930,
"last": 46938,
"maybe_went_rw": 0,
"up": [
4],
"acting": [
95]},
{ "first": 46939,
"last": 46939,
"maybe_went_rw": 0,
"up": [
4],
"acting": []},
{ "first": 46940,
"last": 46955,
"maybe_went_rw": 1,
"up": [
4],
"acting": [
4]},
{ "first": 46956,
"last": 46960,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
86,
4]},
{ "first": 46961,
"last": 46971,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
86,
4,
120]},
{ "first": 46972,
"last": 46972,
"maybe_went_rw": 0,
"up": [
86,
4],
"acting": [
86,
4]},
{ "first": 46973,
"last": 46980,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
86,
4,
120]},
{ "first": 46981,
"last": 46981,
"maybe_went_rw": 0,
"up": [
86,
4],
"acting": [
86,
4]},
{ "first": 46982,
"last": 46993,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
86,
4,
120]},
{ "first": 46994,
"last": 46994,
"maybe_went_rw": 0,
"up": [
86,
4],
"acting": [
86,
4]},
{ "first": 46995,
"last": 47007,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
86,
4,
120]},
{ "first": 47008,
"last": 47020,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
86,
4]},
{ "first": 47021,
"last": 47022,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
86,
4,
120]},
{ "first": 47023,
"last": 47057,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
86]},
{ "first": 47058,
"last": 47058,
"maybe_went_rw": 0,
"up": [
86,
120],
"acting": [
95,
86]},
{ "first": 47059,
"last": 47117,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
86]},
{ "first": 47118,
"last": 47118,
"maybe_went_rw": 0,
"up": [
4,
120],
"acting": [
95]},
{ "first": 47119,
"last": 47120,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
86]},
{ "first": 47121,
"last": 47121,
"maybe_went_rw": 0,
"up": [
86,
4,
120],
"acting": [
86]},
{ "first": 47122,
"last": 47208,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
86]},
{ "first": 47209,
"last": 47217,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
95,
86]},
{ "first": 47218,
"last": 47303,
"maybe_went_rw": 1,
"up": [
86,
4,
120],
"acting": [
95,
86]},
{ "first": 47304,
"last": 47310,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
95,
86]},
{ "first": 47311,
"last": 47317,
"maybe_went_rw": 1,
"up": [
86,
4,
121],
"acting": [
95,
86]},
{ "first": 47318,
"last": 47324,
"maybe_went_rw": 1,
"up": [
86,
4,
122],
"acting": [
95,
86]},
{ "first": 47325,
"last": 47332,
"maybe_went_rw": 1,
"up": [
86,
4,
117],
"acting": [
95,
86]},
{ "first": 47333,
"last": 47338,
"maybe_went_rw": 1,
"up": [
86,
4,
105],
"acting": [
95,
86]},
{ "first": 47339,
"last": 47343,
"maybe_went_rw": 1,
"up": [
86,
4,
127],
"acting": [
95,
86]},
{ "first": 47344,
"last": 47351,
"maybe_went_rw": 1,
"up": [
86,
4,
125],
"acting": [
95,
86]},
{ "first": 47352,
"last": 47357,
"maybe_went_rw": 1,
"up": [
86,
4,
113],
"acting": [
95,
86]},
{ "first": 47358,
"last": 47364,
"maybe_went_rw": 1,
"up": [
86,
4,
126],
"acting": [
95,
86]},
{ "first": 47365,
"last": 47369,
"maybe_went_rw": 1,
"up": [
86,
4,
123],
"acting": [
95,
86]},
{ "first": 47370,
"last": 49743,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
95,
86]},
{ "first": 49744,
"last": 49781,
"maybe_went_rw": 1,
"up": [
86],
"acting": [
95,
86]},
{ "first": 49782,
"last": 50238,
"maybe_went_rw": 1,
"up": [
86,
4],
"acting": [
95,
86]},
{ "first": 50239,
"last": 50248,
"maybe_went_rw": 0,
"up": [
4],
"acting": [
95]},
{ "first": 50249,
"last": 50249,
"maybe_went_rw": 0,
"up": [
4],
"acting": []},
{ "first": 50250,
"last": 50297,
"maybe_went_rw": 1,
"up": [
4],
"acting": [
4]},
{ "first": 50298,
"last": 50340,
"maybe_went_rw": 1,
"up": [
35,
68],
"acting": [
35,
68]},
{ "first": 50341,
"last": 50342,
"maybe_went_rw": 1,
"up": [
72,
4],
"acting": [
72,
4]},
{ "first": 50343,
"last": 52607,
"maybe_went_rw": 1,
"up": [
35,
68],
"acting": [
35,
68]},
{ "first": 52608,
"last": 52621,
"maybe_went_rw": 1,
"up": [
72,
4],
"acting": [
72,
4]},
{ "first": 52622,
"last": 53295,
"maybe_went_rw": 1,
"up": [
35,
68],
"acting": [
35,
68]},
{ "first": 53296,
"last": 53299,
"maybe_went_rw": 1,
"up": [
72,
4],
"acting": [
72,
4]},
{ "first": 53300,
"last": 54996,
"maybe_went_rw": 1,
"up": [
35,
68],
"acting": [
35,
68]},
{ "first": 54997,
"last": 55010,
"maybe_went_rw": 1,
"up": [
72,
4],
"acting": [
72,
4]},
{ "first": 55011,
"last": 60541,
"maybe_went_rw": 1,
"up": [
35,
68],
"acting": [
35,
68]},
{ "first": 60542,
"last": 70919,
"maybe_went_rw": 1,
"up": [
35,
4,
72],
"acting": [
35,
4,
72]},
{ "first": 70920,
"last": 70924,
"maybe_went_rw": 1,
"up": [
35,
4],
"acting": [
35,
4]},
{ "first": 70925,
"last": 81164,
"maybe_went_rw": 1,
"up": [
35,
4,
72],
"acting": [
35,
4,
72]}],
"probing_osds": [
4,
35,
68,
72,
86,
95,
120],
"down_osds_we_would_probe": [],
"peering_blocked_by": []},
{ "name": "Started",
"enter_time": "2013-02-19 11:14:35.094736"}]}
