Dear CEPH ,
Urgent question, I met a "FAILED assert(0 == "unexpected error")" yesterday , Now i have not way to start this OSDS
I have attached my logs in the attachment, and some ceph configurations as below
osd_pool_default_pgp_num = 300
osd_pool_default_size = 2
osd_pool_default_min_size = 1
osd_pool_default_pg_num = 300
mon_host = 10.1.0.213,10.1.0.214
osd_crush_chooseleaf_type = 1
mds_cache_size = 500000
osd objectstore = keyvaluestore-dev
Detailed error information :
-13> 2014-09-05 15:07:35.279863 7f4d988b9700 2 waiting 51 > 50 ops || 11642907 > 104857600
-12> 2014-09-05 15:07:35.279899 7f4d978b7700 2 waiting 51 > 50 ops || 11642899 > 104857600
-11> 2014-09-05 15:07:35.279919 7f4d990ba700 2 waiting 51 > 50 ops || 11642901 > 104857600
-10> 2014-09-05 15:07:35.326803 7f4d9a8bd700 10 monclient: tick
-9> 2014-09-05 15:07:35.326837 7f4d9a8bd700 10 monclient: _check_auth_rotating have uptodate secrets (they expire after 2014-09-05 15:07:05.326835)
-8> 2014-09-05 15:07:35.326871 7f4d9a8bd700 10 monclient: renew subs? (now: 2014-09-05 15:07:35.326871; renew after: 2014-09-05 15:10:02.464341) -- no
-7> 2014-09-05 15:07:35.343657 7f4d978b7700 2 waiting 51 > 50 ops || 11044551 > 104857600
-6> 2014-09-05 15:07:35.343654 7f4e1ee72700 1 -- 10.1.0.221:6801/4013 --> osd.12 10.1.0.219:6810/32654 -- pg_info(1 pgs e1267:0.f1) v4 -- ?+0 0x18dcf000
-5> 2014-09-05 15:07:35.343680 7f4d990ba700 2 waiting 51 > 50 ops || 11044553 > 104857600
-4> 2014-09-05 15:07:35.343686 7f4d988b9700 2 waiting 51 > 50 ops || 11044579 > 104857600
-3> 2014-09-05 15:07:35.344875 7f4e1fe74700 0 error (22) Invalid argument not handled on operation 9 (336.0.3, or op 3, counting from 0)
-2> 2014-09-05 15:07:35.344902 7f4e1fe74700 0 unexpected error code
-1> 2014-09-05 15:07:35.344903 7f4e1fe74700 0 transaction dump:
{ "ops": [
{ "op_num": 0,
"op_name": "remove",
"collection": "0.a9_head",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0"},
{ "op_num": 1,
"op_name": "mkcoll",
"collection": "0.a9_TEMP"},
{ "op_num": 2,
"op_name": "remove",
"collection": "0.a9_TEMP",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0"},
{ "op_num": 3,
"op_name": "touch",
"collection": "0.a9_head",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0"},
{ "op_num": 4,
"op_name": "omap_setheader",
"collection": "0.a9_head",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0",
"header_length": "0"},
{ "op_num": 5,
"op_name": "write",
"collection": "0.a9_head",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0",
"length": 1160,
"offset": 0,
"bufferlist length": 1160},
{ "op_num": 6,
"op_name": "omap_setkeys",
"collection": "0.a9_head",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0",
"attr_lens": {}},
{ "op_num": 7,
"op_name": "setattrs",
"collection": "0.a9_head",
"oid": "4b0fea9\/1000053b885.00000000\/head\/\/0",
"attr_lens": { "_": 239,
"_parent": 250,
"snapset": 31}},
{ "op_num": 8,
"op_name": "omap_setkeys",
"collection": "meta",
"oid": "16ef7597\/infos\/head\/\/-1",
"attr_lens": { "0.a9_epoch": 4,
"0.a9_info": 684}},
{ "op_num": 9,
"op_name": "remove",
"collection": "0.a9_head",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0"},
{ "op_num": 10,
"op_name": "remove",
"collection": "0.a9_TEMP",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0"},
{ "op_num": 11,
"op_name": "touch",
"collection": "0.a9_head",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0"},
{ "op_num": 12,
"op_name": "omap_setheader",
"collection": "0.a9_head",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0",
"header_length": "0"},
{ "op_num": 13,
"op_name": "write",
"collection": "0.a9_head",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0",
"length": 507284,
"offset": 0,
"bufferlist length": 507284},
{ "op_num": 14,
"op_name": "omap_setkeys",
"collection": "0.a9_head",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0",
"attr_lens": {}},
{ "op_num": 15,
"op_name": "setattrs",
"collection": "0.a9_head",
"oid": "4c56f2a9\/10000c04096.00000000\/head\/\/0",
"attr_lens": { "_": 239,
"snapset": 31}},
{ "op_num": 16,
"op_name": "omap_setkeys",
"collection": "meta",
"oid": "16ef7597\/infos\/head\/\/-1",
"attr_lens": { "0.a9_epoch": 4,
"0.a9_info": 684}},
{ "op_num": 17,
"op_name": "remove",
"collection": "0.a9_head",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0"},
{ "op_num": 18,
"op_name": "remove",
"collection": "0.a9_TEMP",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0"},
{ "op_num": 19,
"op_name": "touch",
"collection": "0.a9_head",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0"},
{ "op_num": 20,
"op_name": "omap_setheader",
"collection": "0.a9_head",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0",
"header_length": "0"},
{ "op_num": 21,
"op_name": "write",
"collection": "0.a9_head",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0",
"length": 3495,
"offset": 0,
"bufferlist length": 3495},
{ "op_num": 22,
"op_name": "omap_setkeys",
"collection": "0.a9_head",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0",
"attr_lens": {}},
{ "op_num": 23,
"op_name": "setattrs",
"collection": "0.a9_head",
"oid": "794064a9\/10000c040e0.00000000\/head\/\/0",
"attr_lens": { "_": 239,
"snapset": 31}},
{ "op_num": 24,
"op_name": "omap_setkeys",
"collection": "meta",
"oid": "16ef7597\/infos\/head\/\/-1",
"attr_lens": { "0.a9_epoch": 4,
"0.a9_info": 684}},
{ "op_num": 25,
"op_name": "remove",
"collection": "0.a9_head",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0"},
{ "op_num": 26,
"op_name": "remove",
"collection": "0.a9_TEMP",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0"},
{ "op_num": 27,
"op_name": "touch",
"collection": "0.a9_head",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0"},
{ "op_num": 28,
"op_name": "omap_setheader",
"collection": "0.a9_head",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0",
"header_length": "0"},
{ "op_num": 29,
"op_name": "write",
"collection": "0.a9_head",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0",
"length": 21502,
"offset": 0,
"bufferlist length": 21502},
{ "op_num": 30,
"op_name": "omap_setkeys",
"collection": "0.a9_head",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0",
"attr_lens": {}},
{ "op_num": 31,
"op_name": "setattrs",
"collection": "0.a9_head",
"oid": "b4f73ea9\/10000c04209.00000000\/head\/\/0",
"attr_lens": { "_": 239,
"snapset": 31}},
{ "op_num": 32,
"op_name": "omap_setkeys",
"collection": "meta",
"oid": "16ef7597\/infos\/head\/\/-1",
"attr_lens": { "0.a9_epoch": 4,
"0.a9_info": 684}},
{ "op_num": 33,
"op_name": "remove",
"collection": "0.a9_head",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0"},
{ "op_num": 34,
"op_name": "remove",
"collection": "0.a9_TEMP",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0"},
{ "op_num": 35,
"op_name": "touch",
"collection": "0.a9_head",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0"},
{ "op_num": 36,
"op_name": "omap_setheader",
"collection": "0.a9_head",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0",
"header_length": "0"},
{ "op_num": 37,
"op_name": "write",
"collection": "0.a9_head",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0",
"length": 456155,
"offset": 0,
"bufferlist length": 456155},
{ "op_num": 38,
"op_name": "omap_setkeys",
"collection": "0.a9_head",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0",
"attr_lens": {}},
{ "op_num": 39,
"op_name": "setattrs",
"collection": "0.a9_head",
"oid": "f6ea1ea9\/10000c04238.00000000\/head\/\/0",
"attr_lens": { "_": 239,
"snapset": 31}},
{ "op_num": 40,
"op_name": "omap_setkeys",
"collection": "meta",
"oid": "16ef7597\/infos\/head\/\/-1",
"attr_lens": { "0.a9_epoch": 4,
"0.a9_info": 684}}]}
0> 2014-09-05 15:07:35.346659 7f4e1fe74700 -1 os/KeyValueStore.cc: In function 'unsigned int KeyValueStore::_do_transaction(ObjectStore::Transaction&, KeyValueStore::BufferTransaction&, SequencerPosition&, ThreadPool::TPHandle*)' thread 7f4e1fe74700 time 2014-09-05 15:07:35.345273
os/KeyValueStore.cc: 1500: FAILED assert(0 == "unexpected error")
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.ceph.com/pipermail/ceph-users-ceph.com/attachments/20140905/3bc4c53a/attachment.htm>
