Hi Ben,
At first glance i would say the CPU's are a bit weak for this setup. Recommended is to have at least 1 core per OSD. Since you have 8 cores and 10 OSD's there isn't much left for other processes.
Furthermore, did you upgrade the firmware of those DC S4500's to the latest firmware? (SCV10142)
If not, you can upgrade them via the Intel Data Center upgrade tool: https://downloadcenter.intel.com/download/27863?v=t
Kind regards,
Caspar
2018-07-04 10:26 GMT+02:00 Benjamin Naber <der-coder@xxxxxxxxxxxxxx>:
Hi @all,
im currently in testing for setup an production environment based on the following OSD Nodes:
CEPH Version: luminous 12.2.5
5x OSD Nodes with following specs:
- 8 Core Intel Xeon 2,0 GHZ
- 96GB Ram
- 10x 1,92 TB Intel DC S4500 connectet via SATA
- 4x 10 Gbit NIC 2 bonded via LACP for Backend Network 2 bonded via LACP for Backend Network.
if i run some fio benchmark via a VM that ist running on a RBD Device on a KVM testing Host. the cluster always runs into slow request warning. Also the performance goes heavy down.
If i dump the osd that stucks, i get the following output:
{
"ops": [
{
"description": "osd_op(client.141944.0:359346834 13.1da 13:5b8b7fd3:::rbd_data. 170a3238e1f29. 00000000000000be:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.475879",
"age": 287.180328,
"duration": 287.180355,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346834
},
"events": [
{
"time": "2018-07-04 10:00:49.475879",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.476935",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.477547",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.477578",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.477614",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.484679",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.484681",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485588",
"event": "sub_op_commit_rec from 5"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346835 13.1da 13:5b8b7fd3:::rbd_data. 170a3238e1f29. 00000000000000be:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.477065",
"age": 287.179143,
"duration": 287.179221,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346835
},
"events": [
{
"time": "2018-07-04 10:00:49.477065",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.478116",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.478178",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.478201",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.478232",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.484695",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.484696",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485621",
"event": "sub_op_commit_rec from 5"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346440 13.11d 13:b8afbe4a:::rbd_data. 170a3238e1f29. 000000000000005c:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.091127",
"age": 287.565080,
"duration": 287.565196,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346440
},
"events": [
{
"time": "2018-07-04 10:00:49.091127",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.092764",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.092787",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.093062",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.093107",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.099496",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.099502",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.100772",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346441 13.11d 13:b8afbe4a:::rbd_data. 170a3238e1f29. 000000000000005c:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.092772",
"age": 287.563436,
"duration": 287.563588,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346441
},
"events": [
{
"time": "2018-07-04 10:00:49.092772",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.093766",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.093865",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.093895",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.093937",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.099505",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.099506",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.100788",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346442 13.11d 13:b8afbe4a:::rbd_data. 170a3238e1f29. 000000000000005c:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.093773",
"age": 287.562434,
"duration": 287.562622,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346442
},
"events": [
{
"time": "2018-07-04 10:00:49.093773",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.095388",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.095403",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.095428",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.095466",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.103411",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.103413",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.106365",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346443 13.11d 13:b8afbe4a:::rbd_data. 170a3238e1f29. 000000000000005c:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.095395",
"age": 287.560813,
"duration": 287.561037,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346443
},
"events": [
{
"time": "2018-07-04 10:00:49.095395",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.097194",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.098851",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.098878",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.098914",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.103704",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.103706",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.107064",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346172 13.11d 13:b88a4b5e:::rbd_data. 170a3238e1f29. 000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.836391",
"age": 287.819817,
"duration": 287.820078,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346172
},
"events": [
{
"time": "2018-07-04 10:00:48.836391",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.837419",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.837438",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.837548",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.837605",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847008",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847013",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862005",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346372 13.bc 13:3d131f8b:::rbd_data. 170a3238e1f29. 000000000000004b:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.023513",
"age": 287.632694,
"duration": 287.632990,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346372
},
"events": [
{
"time": "2018-07-04 10:00:49.023513",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.025442",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.025464",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.025796",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.025844",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.033594",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.033600",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.036086",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346173 13.11d 13:b88a4b5e:::rbd_data. 170a3238e1f29. 000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 1048576~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.837425",
"age": 287.818782,
"duration": 287.819113,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346173
},
"events": [
{
"time": "2018-07-04 10:00:48.837425",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.838409",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.838427",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.838463",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.838505",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847328",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847330",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862049",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346373 13.bc 13:3d131f8b:::rbd_data. 170a3238e1f29. 000000000000004b:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.025449",
"age": 287.630759,
"duration": 287.631126,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346373
},
"events": [
{
"time": "2018-07-04 10:00:49.025449",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.027010",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.027030",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.027071",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.027126",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.033603",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.033605",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.036143",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346174 13.11d 13:b88a4b5e:::rbd_data. 170a3238e1f29. 000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 2097152~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.838414",
"age": 287.817793,
"duration": 287.818196,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346174
},
"events": [
{
"time": "2018-07-04 10:00:48.838414",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.839187",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.839203",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.839229",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.839269",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847345",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847347",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862060",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346374 13.bc 13:3d131f8b:::rbd_data. 170a3238e1f29. 000000000000004b:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.027016",
"age": 287.629192,
"duration": 287.629634,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346374
},
"events": [
{
"time": "2018-07-04 10:00:49.027016",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.027972",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.027987",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.028017",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.028053",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.033606",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.033609",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.036159",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346175 13.11d 13:b88a4b5e:::rbd_data. 170a3238e1f29. 000000000000001a:head [set-alloc-hint object_size 4194304 write_size 4194304,write 3145728~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:48.839192",
"age": 287.817015,
"duration": 287.817491,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346175
},
"events": [
{
"time": "2018-07-04 10:00:48.839192",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:48.840612",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:48.840626",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:48.840658",
"event": "started"
},
{
"time": "2018-07-04 10:00:48.840693",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:48.847361",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:48.847363",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:48.862089",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346375 13.bc 13:3d131f8b:::rbd_data. 170a3238e1f29. 000000000000004b:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.027978",
"age": 287.628229,
"duration": 287.628739,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346375
},
"events": [
{
"time": "2018-07-04 10:00:49.027978",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.028846",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.028863",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.028889",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.028924",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.035948",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.035950",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.038542",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346792 13.bc 13:3d313592:::rbd_data. 170a3238e1f29. 00000000000000b4:head [write 0~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.423998",
"age": 287.232209,
"duration": 287.232753,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346792
},
"events": [
{
"time": "2018-07-04 10:00:49.423998",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.425331",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.425350",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.425655",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.425715",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.434856",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.434869",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.436410",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346793 13.bc 13:3d313592:::rbd_data. 170a3238e1f29. 00000000000000b4:head [write 1048576~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.425349",
"age": 287.230859,
"duration": 287.231436,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346793
},
"events": [
{
"time": "2018-07-04 10:00:49.425349",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.426349",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.426877",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.427022",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.427112",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.434876",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.434877",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.436661",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346794 13.bc 13:3d313592:::rbd_data. 170a3238e1f29. 00000000000000b4:head [write 2097152~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.426356",
"age": 287.229852,
"duration": 287.230461,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346794
},
"events": [
{
"time": "2018-07-04 10:00:49.426356",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.427126",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.427729",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.427771",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.427812",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.434878",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.434879",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.436686",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346795 13.bc 13:3d313592:::rbd_data. 170a3238e1f29. 00000000000000b4:head [write 3145728~1048576] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.427141",
"age": 287.229066,
"duration": 287.229715,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346795
},
"events": [
{
"time": "2018-07-04 10:00:49.427141",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.428212",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.431693",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.431730",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.431794",
"event": "waiting for subops from 21,26"
},
{
"time": "2018-07-04 10:00:49.437279",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.437282",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.442448",
"event": "sub_op_commit_rec from 21"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346832 13.1da 13:5b8b7fd3:::rbd_data. 170a3238e1f29. 00000000000000be:head [write 0~901120] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.473234",
"age": 287.182974,
"duration": 287.183657,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346832
},
"events": [
{
"time": "2018-07-04 10:00:49.473234",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.474116",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.474135",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.474444",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.474489",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.478163",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.478170",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485418",
"event": "sub_op_commit_rec from 5"
}
]
}
},
{
"description": "osd_op(client.141944.0:359346833 13.1da 13:5b8b7fd3:::rbd_data. 170a3238e1f29. 00000000000000be:head [write 901120~1196032] snapc 0=[] ondisk+write+known_if_ redirected e2755)",
"initiated_at": "2018-07-04 10:00:49.474651",
"age": 287.181557,
"duration": 287.182274,
"type_data": {
"flag_point": "waiting for sub ops",
"client_info": {
"client": "client.141944",
"client_addr": "10.111.90.1:0/3532639465",
"tid": 359346833
},
"events": [
{
"time": "2018-07-04 10:00:49.474651",
"event": "initiated"
},
{
"time": "2018-07-04 10:00:49.475829",
"event": "queued_for_pg"
},
{
"time": "2018-07-04 10:00:49.475845",
"event": "reached_pg"
},
{
"time": "2018-07-04 10:00:49.475881",
"event": "started"
},
{
"time": "2018-07-04 10:00:49.475923",
"event": "waiting for subops from 5,26"
},
{
"time": "2018-07-04 10:00:49.484417",
"event": "op_commit"
},
{
"time": "2018-07-04 10:00:49.484420",
"event": "op_applied"
},
{
"time": "2018-07-04 10:00:49.485539",
"event": "sub_op_commit_rec from 5"
}
]
}
}
],
"num_ops": 20
}
can someone explain me how i can find the source of the Problem ?
kind regards
Ben
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph. com
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
