Hi @all,
im currently in testing for setup an production environment based on the following OSD Nodes:
CEPH Version: luminous 12.2.5
5x OSD Nodes with following specs:
- 8 Core Intel Xeon 2,0 GHZ
- 96GB Ram
- 10x 1,92 TB Intel DC S4500 connectet via SATA
- 4x 10 Gbit NIC 2 bonded via LACP for Backend Network 2 bonded via LACP for Backend Network.
if i run some fio benchmark via a VM that ist running on a RBD Device on a KVM testing Host. the cluster always runs into slow request warning. Also the performance goes heavy down.
If i dump the osd that stucks, i get the following output:
{
"ops": [
{
"description": "osd_op(client.141944.0:359346834 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.475879",
"age": 287.180328,
"duration": 287.180355,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346834
},
"events": [
{
"time": "2018-07-04 10:00:49.475879",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.476935",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.477547",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.477578",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.477614",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.484679",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.484681",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485588",
"event": "sub_op_commit_rec from 5"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346835 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.477065",
"age": 287.179143,
"duration": 287.179221,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346835
},
"events": [
{
"time": "2018-07-04 10:00:49.477065",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.478116",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.478178",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.478201",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.478232",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.484695",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.484696",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485621",
"event": "sub_op_commit_rec from 5"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346440 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.091127",
"age": 287.565080,
"duration": 287.565196,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346440
},
"events": [
{
"time": "2018-07-04 10:00:49.091127",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.092764",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.092787",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.093062",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.093107",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.099496",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.099502",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.100772",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346441 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.092772",
"age": 287.563436,
"duration": 287.563588,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346441
},
"events": [
{
"time": "2018-07-04 10:00:49.092772",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.093766",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.093865",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.093895",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.093937",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.099505",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.099506",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.100788",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346442 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.093773",
"age": 287.562434,
"duration": 287.562622,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346442
},
"events": [
{
"time": "2018-07-04 10:00:49.093773",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.095388",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.095403",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.095428",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.095466",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.103411",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.103413",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.106365",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346443 13.11d 13:b8afbe4a:::rbd_data.170a3238e1f29.000000000000005c:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.095395",
"age": 287.560813,
"duration": 287.561037,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346443
},
"events": [
{
"time": "2018-07-04 10:00:49.095395",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.097194",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.098851",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.098878",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.098914",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.103704",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.103706",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.107064",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346172 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.836391",
"age": 287.819817,
"duration": 287.820078,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346172
},
"events": [
{
"time": "2018-07-04 10:00:48.836391",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.837419",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.837438",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.837548",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.837605",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847008",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847013",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862005",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346372 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.023513",
"age": 287.632694,
"duration": 287.632990,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346372
},
"events": [
{
"time": "2018-07-04 10:00:49.023513",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.025442",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.025464",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.025796",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.025844",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.033594",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.033600",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.036086",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346173 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.837425",
"age": 287.818782,
"duration": 287.819113,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346173
},
"events": [
{
"time": "2018-07-04 10:00:48.837425",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.838409",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.838427",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.838463",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.838505",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847328",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847330",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862049",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346373 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.025449",
"age": 287.630759,
"duration": 287.631126,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346373
},
"events": [
{
"time": "2018-07-04 10:00:49.025449",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.027010",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.027030",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.027071",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.027126",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.033603",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.033605",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.036143",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346174 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.838414",
"age": 287.817793,
"duration": 287.818196,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346174
},
"events": [
{
"time": "2018-07-04 10:00:48.838414",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.839187",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.839203",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.839229",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.839269",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847345",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847347",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862060",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346374 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.027016",
"age": 287.629192,
"duration": 287.629634,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346374
},
"events": [
{
"time": "2018-07-04 10:00:49.027016",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.027972",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.027987",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.028017",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.028053",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.033606",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.033609",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.036159",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346175 13.11d 13:b88a4b5e:::rbd_data.170a3238e1f29.000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.839192",
"age": 287.817015,
"duration": 287.817491,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346175
},
"events": [
{
"time": "2018-07-04 10:00:48.839192",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.840612",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.840626",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.840658",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.840693",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847361",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847363",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862089",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346375 13.bc 13:3d131f8b:::rbd_data.170a3238e1f29.000000000000004b:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.027978",
"age": 287.628229,
"duration": 287.628739,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346375
},
"events": [
{
"time": "2018-07-04 10:00:49.027978",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.028846",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.028863",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.028889",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.028924",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.035948",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.035950",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.038542",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346792 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.423998",
"age": 287.232209,
"duration": 287.232753,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346792
},
"events": [
{
"time": "2018-07-04 10:00:49.423998",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.425331",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.425350",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.425655",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.425715",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.434856",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.434869",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.436410",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346793 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.425349",
"age": 287.230859,
"duration": 287.231436,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346793
},
"events": [
{
"time": "2018-07-04 10:00:49.425349",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.426349",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.426877",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.427022",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.427112",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.434876",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.434877",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.436661",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346794 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.426356",
"age": 287.229852,
"duration": 287.230461,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346794
},
"events": [
{
"time": "2018-07-04 10:00:49.426356",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.427126",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.427729",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.427771",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.427812",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.434878",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.434879",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.436686",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346795 13.bc 13:3d313592:::rbd_data.170a3238e1f29.00000000000000b4:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.427141",
"age": 287.229066,
"duration": 287.229715,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346795
},
"events": [
{
"time": "2018-07-04 10:00:49.427141",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.428212",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.431693",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.431730",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.431794",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.437279",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.437282",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.442448",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346832 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 0~901120] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.473234",
"age": 287.182974,
"duration": 287.183657,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346832
},
"events": [
{
"time": "2018-07-04 10:00:49.473234",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.474116",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.474135",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.474444",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.474489",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.478163",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.478170",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485418",
"event": "sub_op_commit_rec from 5"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346833 13.1da 13:5b8b7fd3:::rbd_data.170a3238e1f29.00000000000000be:head [write 901120~1196032] snapc 0=[] ondisk+write+known_if_redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.474651",
"age": 287.181557,
"duration": 287.182274,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346833
},
"events": [
{
"time": "2018-07-04 10:00:49.474651",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.475829",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.475845",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.475881",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.475923",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.484417",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.484420",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485539",
"event": "sub_op_commit_rec from 5"
}
]
}
}
],
"num_ops": 20
}
can someone explain me how i can find the source of the Problem ?
kind regards
Ben
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
