Looks like you didn't tell the whole story, please post the *full* output of ceph -s and ceph osd df tree. Wild guess: you need to increase "mon max pg per osd" Paul -- Paul Emmerich Looking for help with your Ceph cluster? Contact us at https://croit.io croit GmbH Freseniusstr. 31h 81247 München www.croit.io Tel: +49 89 1896585 90 On Thu, Oct 31, 2019 at 8:17 PM Thomas <74cmonty@xxxxxxxxx> wrote: > > This is the output of OSD.270 that remains with slow requests blocked > even after restarting. > What's the interpretation of it? > > root@ld5507:~# ceph daemon osd.270 dump_blocked_ops [330/1857] > { > "ops": [ > { > "description": "osd_pg_create(e293649 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:22:13.563017", > "age": 2785.2698560419999, > "duration": 2785.2699056299998, > "type_data": { > "flag_point": "started", > "events": [ > { > "time": "2019-10-31 19:22:13.563017", > "event": "initiated" > }, > { > "time": "2019-10-31 19:22:13.563017", > "event": "header_read" > }, > { > "time": "2019-10-31 19:22:13.563011", > "event": "throttled" > }, > { > "time": "2019-10-31 19:22:13.563024", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.881441", > "event": "dispatched" > }, [300/1857] > { > "time": "2019-10-31 20:07:43.881472", > "event": "wait for new map" > }, > { > "time": "2019-10-31 20:07:44.665714", > "event": "started" > } > ] > } > }, > { > "description": "osd_pg_create(e293650 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:23:16.150040", > "age": 2722.6828331659999, > "duration": 2722.683007228, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:23:16.150040", > "event": "initiated" > }, > { > "time": "2019-10-31 19:23:16.150040", > "event": "header_read" > }, > { > "time": "2019-10-31 19:23:16.150035", > "event": "throttled" > }, [269/1857] > { > "time": "2019-10-31 19:23:16.150055", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.882197", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.882198", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293651 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:23:17.779034", > "age": 2721.0538393319998, > "duration": 2721.0541152350002, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:23:17.779034", > "event": "initiated" > }, > { > "time": "2019-10-31 19:23:17.779034", > "event": "header_read" > }, [238/1857] > { > "time": "2019-10-31 19:23:17.779027", > "event": "throttled" > }, > { > "time": "2019-10-31 19:23:17.779044", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.882326", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.882328", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293652 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:23:22.178466", > "age": 2716.6544071140002, > "duration": 2716.654784073, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:23:22.178466", > "event": "initiated" > }, [207/1857] > { > "time": "2019-10-31 19:23:22.178466", > "event": "header_read" > }, > { > "time": "2019-10-31 19:23:22.178463", > "event": "throttled" > }, > { > "time": "2019-10-31 19:23:22.178471", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.882355", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.882356", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293653 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:23:28.227563", > "age": 2710.6053095460002, > "duration": 2710.6057835810002, > "type_data": { > "flag_point": "delayed", > "events": [ [176/1857] > { > "time": "2019-10-31 19:23:28.227563", > "event": "initiated" > }, > { > "time": "2019-10-31 19:23:28.227563", > "event": "header_read" > }, > { > "time": "2019-10-31 19:23:28.227559", > "event": "throttled" > }, > { > "time": "2019-10-31 19:23:28.227570", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.882383", > "event": "dispatched" > }, "age": 2574.9672041270001, [145/1857] > "duration": 2574.9677752090001, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:25:43.865669", > "event": "initiated" > }, > { > "time": "2019-10-31 19:25:43.865669", > "event": "header_read" > }, > { > "time": "2019-10-31 19:25:43.865663", > "event": "throttled" > }, > { > "time": "2019-10-31 19:25:43.865682", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.883843", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.883846", > "event": "wait for new map" > } [117/1857] > ] > } > }, > { > "description": "osd_pg_create(e293655 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:26:16.129187", > "age": 2542.703685858, > "duration": 2542.7043550879998, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:26:16.129187", > "event": "initiated" > }, > { > "time": "2019-10-31 19:26:16.129187", > "event": "header_read" > }, > { > "time": "2019-10-31 19:26:16.129181", > "event": "throttled" > }, > { > "time": "2019-10-31 19:26:16.129197", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.883928", > "event": "dispatched" > }, [86/1857] > { > "time": "2019-10-31 20:07:43.883929", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293656 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:30:47.890777", > "age": 2270.9420963450002, > "duration": 2270.9428696479999, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:30:47.890777", > "event": "initiated" > }, > { > "time": "2019-10-31 19:30:47.890777", > "event": "header_read" > }, > { > "time": "2019-10-31 19:30:47.890772", > "event": "throttled" > }, > { > "time": "2019-10-31 19:30:47.890788", > "event": "all_read" > }, [55/1857] > { > "time": "2019-10-31 20:07:43.913783", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.913786", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293657 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:35:32.323775", > "age": 1986.5090976690001, > "duration": 1986.509968458, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:35:32.323775", > "event": "initiated" > }, > { > "time": "2019-10-31 19:35:32.323775", > "event": "header_read" > }, > { > "time": "2019-10-31 19:35:32.323769", > "event": "throttled" > }, [24/1857] > { > "time": "2019-10-31 19:35:32.323802", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.917500", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.917503", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293658 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:36:32.783376", > "age": 1926.0494967300001, > "duration": 1926.050464765, > "type_data": { > "flag_point": "delayed", > "events": [ > { > "time": "2019-10-31 19:36:32.783376", > "event": "initiated" > }, > { > "time": "2019-10-31 19:36:32.783376", > "event": "header_read" > }, > { > "time": "2019-10-31 19:36:32.783373", > "event": "throttled" > }, > { > "time": "2019-10-31 19:36:32.783388", > "event": "all_read" > }, > { > "time": "2019-10-31 20:07:43.919280", > "event": "dispatched" > }, > { > "time": "2019-10-31 20:07:43.919282", > "event": "wait for new map" > } > ] > } > } > ], > "complaint_time": 30, > "num_blocked_ops": 10 > } > > { > "time": "2019-10-31 20:07:43.882384", > "event": "wait for new map" > } > ] > } > }, > { > "description": "osd_pg_create(e293654 59.b:267033 > 59.2c:267033)", > "initiated_at": "2019-10-31 19:25:43.865669", > > > Am 31.10.2019 um 18:27 schrieb Paul Emmerich: > > ceph daemon osd.X dump_blocked_ops > _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |