Hi Team,
In one of our ceph cluster we observe that there are many slow IOPS in all our OSD's and most of the latency is happening between two
set of operations which are shown below.
{
"time": "2019-11-12 08:29:58.128669",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484235",
"event": "commit_sent"
"time": "2019-11-12 08:29:58.128669",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484235",
"event": "commit_sent"
Is there any way to know what is causing this issue and rectify this problem? As you can see in the above output of historical_slow_ops there is a 10 second delay between the two operations which is impacting the performance. Attaching one of the OSD historic slow ops output to this mail for reference.
{
"num to keep": 20,
"threshold to keep": 10,
"Ops": [
{
"description": "osd_op(client.71027356.0:194654 1.7s0 1:e005f8bb:::%2fc20%2fvx0000039%2fpot%2fsridhar_106%2fhls%2fh_63538bf124417ce82e34317a6c95097a%2fvar2493000%2fseg20271245_w1573279521.ts:head [delete] snapc 0=[] ondisk+write+peerstat_old+known_if_redirected e30337)",
"initiated_at": "2019-11-12 08:29:50.410364",
"age": 77561.712867374998,
"duration": 12.158651442,
"type_data": {
"flag_point": "commit sent; apply or cleanup",
"client_info": {
"client": "client.71027356",
"client_addr": "10.50.62.163:0/478395830",
"tid": 194654
},
"events": [
{
"time": "2019-11-12 08:29:50.410364",
"event": "initiated"
},
{
"time": "2019-11-12 08:29:50.410364",
"event": "header_read"
},
{
"time": "2019-11-12 08:29:50.410361",
"event": "throttled"
},
{
"time": "2019-11-12 08:29:50.410368",
"event": "all_read"
},
{
"time": "2019-11-12 08:29:50.410369",
"event": "dispatched"
},
{
"time": "2019-11-12 08:29:50.410374",
"event": "queued_for_pg"
},
{
"time": "2019-11-12 08:29:50.410392",
"event": "reached_pg"
},
{
"time": "2019-11-12 08:29:50.410562",
"event": "started"
},
{
"time": "2019-11-12 08:29:50.410684",
"event": "sub_op_started"
},
{
"time": "2019-11-12 08:29:50.419974",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:02.568999",
"event": "commit_sent"
},
{
"time": "2019-11-12 08:30:02.569015",
"event": "done"
}
]
}
},
{
"description": "osd_op(client.71027356.0:195247 1.7s0 1:e0013cdb:::%2fc20%2fvx0000039%2fpot%2fsridhar_84%2fhls%2fh_598d9980480259996ca22bd53af09f02%2fvar370000%2fseg20315902_w1573547390.ts:head [writefull 0~220148,setxattr mode (7),setxattr uid (5),setxattr gid (5),setxattr size (7),setxattr mtime (11),setxattr xattr (2),setxattr meal_flags (3),setxattr expire (7),setxattr md5 (33)] snapc 0=[] ondisk+write+known_if_redirected e30337)",
"initiated_at": "2019-11-12 08:29:56.741549",
"age": 77555.381682584004,
"duration": 11.742638746000001,
"type_data": {
"flag_point": "commit sent; apply or cleanup",
"client_info": {
"client": "client.71027356",
"client_addr": "10.50.62.163:0/478395830",
"tid": 195247
},
"events": [
{
"time": "2019-11-12 08:29:56.741549",
"event": "initiated"
},
{
"time": "2019-11-12 08:29:56.741549",
"event": "header_read"
},
{
"time": "2019-11-12 08:29:56.740440",
"event": "throttled"
},
{
"time": "2019-11-12 08:29:56.741553",
"event": "all_read"
},
{
"time": "2019-11-12 08:29:56.741555",
"event": "dispatched"
},
{
"time": "2019-11-12 08:29:56.741559",
"event": "queued_for_pg"
},
{
"time": "2019-11-12 08:29:56.741578",
"event": "reached_pg"
},
{
"time": "2019-11-12 08:29:56.741735",
"event": "started"
},
{
"time": "2019-11-12 08:29:56.742213",
"event": "sub_op_started"
},
{
"time": "2019-11-12 08:29:56.790062",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484174",
"event": "commit_sent"
},
{
"time": "2019-11-12 08:30:08.484187",
"event": "done"
}
]
}
},
{
"description": "osd_op(client.71146410.0:90943 1.7s0 1:e0001313:::%2fc20%2fvx0000039%2fpot%2fsridhar_131%2fhls%2fh_74ee3ac80a0176f427a8ee7f2a9f922b%2fvar1058000%2fseg20271014_w1573280004.ts:head [delete] snapc 0=[] ondisk+write+peerstat_old+known_if_redirected e30337)",
"initiated_at": "2019-11-12 08:29:58.109953",
"age": 77554.013277881997,
"duration": 10.374296185,
"type_data": {
"flag_point": "commit sent; apply or cleanup",
"client_info": {
"client": "client.71146410",
"client_addr": "10.50.62.164:0/3526276795",
"tid": 90943
},
"events": [
{
"time": "2019-11-12 08:29:58.109953",
"event": "initiated"
},
{
"time": "2019-11-12 08:29:58.109953",
"event": "header_read"
},
{
"time": "2019-11-12 08:29:58.109950",
"event": "throttled"
},
{
"time": "2019-11-12 08:29:58.109957",
"event": "all_read"
},
{
"time": "2019-11-12 08:29:58.109958",
"event": "dispatched"
},
{
"time": "2019-11-12 08:29:58.109962",
"event": "queued_for_pg"
},
{
"time": "2019-11-12 08:29:58.109978",
"event": "reached_pg"
},
{
"time": "2019-11-12 08:29:58.110122",
"event": "started"
},
{
"time": "2019-11-12 08:29:58.110241",
"event": "sub_op_started"
},
{
"time": "2019-11-12 08:29:58.128669",
"event": "sub_op_committed"
},
{
"time": "2019-11-12 08:30:08.484235",
"event": "commit_sent"
},
{
"time": "2019-11-12 08:30:08.484249",
"event": "done"
}
]
}
}
]
}
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
