On Wed, Jul 4, 2018 at 6:26 PM, Benjamin Naber <der-coder@xxxxxxxxxxxxxx> wrote:
> Hi @all,
>
> im currently in testing for setup an production environment based on the following OSD Nodes:
>
> CEPH Version: luminous 12.2.5
>
> 5x OSD Nodes with following specs:
>
> - 8 Core Intel Xeon 2,0 GHZ
>
> - 96GB Ram
>
> - 10x 1,92 TB Intel DC S4500 connectet via SATA
>
> - 4x 10 Gbit NIC 2 bonded via LACP for Backend Network 2 bonded via LACP for Backend Network.
>
> if i run some fio benchmark via a VM that ist running on a RBD Device on a KVM testing Host. the cluster always runs into slow request warning. Also the performance goes heavy down.
>
> If i dump the osd that stucks, i get the following output:
>
> {
> "ops": [
> {
> "description": "osd_op(client.141944.0:359346834 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.475879",
> "age": 287.180328,
> "duration": 287.180355,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346834
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.475879",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.476935",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.477547",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.477578",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.477614",
> "event": "waiting for subops from 5,26"
> },
> {
> "time": "2018-07-04 10:00:49.484679",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.484681",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.485588",
> "event": "sub_op_commit_rec from 5"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346835 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.477065",
> "age": 287.179143,
> "duration": 287.179221,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346835
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.477065",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.478116",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.478178",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.478201",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.478232",
> "event": "waiting for subops from 5,26"
> },
> {
> "time": "2018-07-04 10:00:49.484695",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.484696",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.485621",
> "event": "sub_op_commit_rec from 5"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346440 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.091127",
> "age": 287.565080,
> "duration": 287.565196,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346440
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.091127",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.092764",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.092787",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.093062",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.093107",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.099496",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.099502",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.100772",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346441 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.092772",
> "age": 287.563436,
> "duration": 287.563588,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346441
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.092772",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.093766",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.093865",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.093895",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.093937",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.099505",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.099506",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.100788",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346442 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.093773",
> "age": 287.562434,
> "duration": 287.562622,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346442
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.093773",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.095388",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.095403",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.095428",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.095466",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.103411",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.103413",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.106365",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346443 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.095395",
> "age": 287.560813,
> "duration": 287.561037,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346443
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.095395",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.097194",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.098851",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.098878",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.098914",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.103704",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.103706",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.107064",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346172 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:48.836391",
> "age": 287.819817,
> "duration": 287.820078,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346172
> },
> "events": [
> {
> "time": "2018-07-04 10:00:48.836391",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:48.837419",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:48.837438",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:48.837548",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:48.837605",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:48.847008",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:48.847013",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:48.862005",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346372 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.023513",
> "age": 287.632694,
> "duration": 287.632990,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346372
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.023513",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.025442",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.025464",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.025796",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.025844",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.033594",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.033600",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.036086",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346173 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:48.837425",
> "age": 287.818782,
> "duration": 287.819113,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346173
> },
> "events": [
> {
> "time": "2018-07-04 10:00:48.837425",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:48.838409",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:48.838427",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:48.838463",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:48.838505",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:48.847328",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:48.847330",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:48.862049",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346373 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.025449",
> "age": 287.630759,
> "duration": 287.631126,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346373
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.025449",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.027010",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.027030",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.027071",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.027126",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.033603",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.033605",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.036143",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346174 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:48.838414",
> "age": 287.817793,
> "duration": 287.818196,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346174
> },
> "events": [
> {
> "time": "2018-07-04 10:00:48.838414",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:48.839187",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:48.839203",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:48.839229",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:48.839269",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:48.847345",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:48.847347",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:48.862060",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346374 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.027016",
> "age": 287.629192,
> "duration": 287.629634,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346374
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.027016",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.027972",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.027987",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.028017",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.028053",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.033606",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.033609",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.036159",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346175 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:48.839192",
> "age": 287.817015,
> "duration": 287.817491,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346175
> },
> "events": [
> {
> "time": "2018-07-04 10:00:48.839192",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:48.840612",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:48.840626",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:48.840658",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:48.840693",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:48.847361",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:48.847363",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:48.862089",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346375 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.027978",
> "age": 287.628229,
> "duration": 287.628739,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346375
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.027978",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.028846",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.028863",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.028889",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.028924",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.035948",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.035950",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.038542",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346792 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.423998",
> "age": 287.232209,
> "duration": 287.232753,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346792
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.423998",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.425331",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.425350",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.425655",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.425715",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.434856",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.434869",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.436410",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346793 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.425349",
> "age": 287.230859,
> "duration": 287.231436,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346793
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.425349",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.426349",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.426877",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.427022",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.427112",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.434876",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.434877",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.436661",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346794 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.426356",
> "age": 287.229852,
> "duration": 287.230461,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346794
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.426356",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.427126",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.427729",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.427771",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.427812",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.434878",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.434879",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.436686",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346795 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.427141",
> "age": 287.229066,
> "duration": 287.229715,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346795
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.427141",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.428212",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.431693",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.431730",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.431794",
> "event": "waiting for subops from 21,26"
> },
> {
> "time": "2018-07-04 10:00:49.437279",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.437282",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.442448",
> "event": "sub_op_commit_rec from 21"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346832 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 0~901120] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.473234",
> "age": 287.182974,
> "duration": 287.183657,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346832
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.473234",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.474116",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.474135",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.474444",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.474489",
> "event": "waiting for subops from 5,26"
> },
> {
> "time": "2018-07-04 10:00:49.478163",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.478170",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.485418",
> "event": "sub_op_commit_rec from 5"
> }
> ]
> }
> },
> {
> "description": "osd_op(client.141944.0:359346833 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 901120~1196032] snapc 0=[] ondisk+write+known_if_redirected e2755)",
> "initiated_at": "2018-07-04 10:00:49.474651",
> "age": 287.181557,
> "duration": 287.182274,
> "type_data": {
> "flag_point": "waiting for sub ops",
> "client_info": {
> "client": "client.141944",
> "client_addr": "10.111.90.1:0/3532639465",
> "tid": 359346833
> },
> "events": [
> {
> "time": "2018-07-04 10:00:49.474651",
> "event": "initiated"
> },
> {
> "time": "2018-07-04 10:00:49.475829",
> "event": "queued_for_pg"
> },
> {
> "time": "2018-07-04 10:00:49.475845",
> "event": "reached_pg"
> },
> {
> "time": "2018-07-04 10:00:49.475881",
> "event": "started"
> },
> {
> "time": "2018-07-04 10:00:49.475923",
> "event": "waiting for subops from 5,26"
> },
> {
> "time": "2018-07-04 10:00:49.484417",
> "event": "op_commit"
> },
> {
> "time": "2018-07-04 10:00:49.484420",
> "event": "op_applied"
> },
> {
> "time": "2018-07-04 10:00:49.485539",
> "event": "sub_op_commit_rec from 5"
> }
> ]
> }
> }
> ],
> "num_ops": 20
> }
>
> can someone explain me how i can find the source of the Problem ?
All of these are waiting for subops to complete on osd 26.
>
> kind regards
>
> Ben
> _______________________________________________
> ceph-users mailing list
> ceph-users@xxxxxxxxxxxxxx
> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
--
Cheers,
Brad
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
