|
Hi Everyone,
I have been seeing some weird results with rbd bench-write when the number of threads in my tests goes above 32. Specifically with 64 and 128 threads respectively.
I am running an older dumpling release 0.67.4 and am using it to build a set test cases that I will run for testing future releases in an automated fashion.
I see a bug-fixes for bench-write v0.75 —> rbd: fix bench-write command (Hoamai Wang) … I am wondering if related? See sample test output below.
======= NORMAL TEST ==========
[root@node-27 ceph-test]# rbd bench-write volumes/loadtest-rbd0 --io-size 1048576 --io-threads 16 --io-pattern rand
bench-write io_size 1048576 io_threads 16 bytes 1073741824 pattern rand
SEC OPS OPS/SEC BYTES/SEC
1 376 375.65 266087649.43
2 652 325.16 261994438.30
3 908 302.63 250578341.71
4 1148 282.90 230236366.30
5 1340 267.68 205484358.74
elapsed: 5 ops: 1446 ops/sec: 253.19 bytes/sec: 188009371.21
======= WEIRD TEST ==========
[root@node-27 ceph-test]# rbd bench-write volumes/loadtest-rbd0 --io-size 1048576 --io-threads 128 --io-pattern rand
bench-write io_size 1048576 io_threads 128 bytes 1073741824 pattern rand
SEC OPS OPS/SEC BYTES/SEC
1 384 337.12 16194736416915101696.00
2 512 237.67 8562842898446498816.00
3 768 210.92 5066194600812718080.00
4 1024 220.11 3965174267083614720.00
5 1152 216.40 3465237360609427968.00
6 1280 207.23 2986498010687531008.00
7 1664 226.84 2514715944203117568.00
8 1920 236.36 2270865716147703552.00
9 2048 227.05 2045053674517228032.00
10 2560 243.67 1755846484859866880.00
11 2688 244.13 1675406116883640832.00
12 3072 244.41 1467639564426388224.00
13 3200 241.14 1390071006629100032.00
14 3456 237.17 1265939868813378048.00
15 3712 239.56 1190468048400115200.00
16 3968 241.58 1123095458508870016.00
17 4096 238.15 1072546417833582464.00
18 4352 240.01 1017316350597379712.00
19 4608 237.13 949293044175138048.00
20 4864 235.47 893029710006287104.00
21 4992 233.67 863485082645484416.00
22 5120 228.48 823195784066028544.00
23 5376 227.38 780205656792357120.00
24 5504 228.95 767317154779148672.00
25 5760 227.14 727429806747233536.00
26 5888 225.44 706276706322019200.00
27 6144 225.54 677157401345555328.00
28 6400 225.71 650568424673900544.00
29 6656 222.86 617636389350426112.00
30 6784 224.50 610458017766997504.00
31 7040 225.55 591007193895404160.00
32 7296 223.37 564746325168071616.00
33 7424 223.78 556035835815750208.00
34 7680 224.75 539834468179971648.00
35 8064 226.05 517098854196574400.00
36 8192 225.39 507543316869586176.00
37 8448 228.30 498511095941450432.00
38 8832 229.66 479673543746555520.00
39 8960 228.84 471124583205229952.00
40 9216 229.27 458905121166883328.00
41 9472 229.61 447168385720164864.00
42 9856 233.47 436964214525758848.00
43 10112 232.68 424467809770207168.00
44 10240 231.77 417514336196732224.00
45 10496 231.86 407500779715553088.00
46 10624 230.08 399498013584861376.00
47 11008 233.35 391043279738133888.00
48 11264 233.32 382100687976533056.00
49 11520 233.22 373457549496762816.00
50 11648 232.21 367741508793568768.00
51 11904 232.51 360309335406545792.00
52 12160 231.35 350953792015869248.00
53 12416 231.16 343446731043054272.00
54 12544 232.01 341185266978134592.00
55 12928 233.29 332877335529639168.00
56 13056 233.04 329262062622102400.00
57 13312 232.87 322687293406453696.00
58 13568 233.02 316810684805172736.00
59 13824 231.78 309285929137423680.00
60 13952 231.71 306353049821044608.00
61 14080 230.75 302315372155774272.00
62 14336 230.36 296416568090511744.00
63 14720 232.43 291281463002263104.00
64 14976 231.17 284743487487861024.00
65 15104 231.47 282692275460864640.00
66 15232 229.77 278265363060257824.00
67 15488 230.50 274528806424432416.00
68 15744 230.23 269747556006066848.00
69 16000 230.65 265925761494344000.00
70 16256 230.28 261311480178859008.00
71 16384 229.61 0.00
72 16512 229.14 14551.09
73 16768 229.36 43028.51
74 17024 228.25 70294.65
75 17280 228.57 97091.77
76 17408 228.98 110341.44
77 17664 228.38 135571.89
78 17920 227.80 159954.55
79 18048 228.21 172363.58
80 18304 228.74 196556.55
81 18688 229.10 231384.73
82 18816 228.64 242092.45
83 19072 228.42 263729.23
84 19200 228.02 273963.83
85 19584 227.88 305036.32
86 19712 227.68 314893.62
87 19968 227.62 334688.25
88 20096 228.24 345363.62
89 20352 228.32 364666.47
90 20608 227.56 382101.43
91 20736 227.62 391352.70
92 20992 227.73 409520.09
93 21248 228.11 427772.15
94 21504 228.48 445635.29
95 21760 228.64 462738.06
96 22016 227.74 477265.49
97 22144 228.10 486046.51
98 22528 228.85 511293.57
99 22784 228.29 525327.89
100 22912 228.30 532866.01
101 23040 227.98 539528.40
102 23296 227.71 553476.20
103 23552 227.53 567271.58
104 23808 227.74 581768.93
105 23936 227.45 587865.58
106 24192 227.55 601640.06
107 24448 228.14 616455.75
108 24704 228.22 629640.52
109 24960 228.64 643536.23
110 25216 228.66 656101.70
111 25472 228.75 668580.33
112 25728 229.32 682261.82
113 25984 229.42 694358.13
114 26240 228.87 704216.95
115 26368 228.79 709654.51
116 26624 229.04 721660.58
117 26880 229.38 733729.20
118 27136 229.95 746378.14
119 27520 230.40 763762.04
120 27648 229.77 766839.23
121 27904 230.15 778362.07
122 28160 229.91 787612.98
123 28288 229.75 792024.74
124 28544 229.80 801978.50
125 28800 229.91 811977.44
126 29056 229.46 819787.11
127 29184 229.72 825395.11
128 29568 230.42 841640.06
129 29824 230.76 851908.57
130 30080 231.30 862740.18
131 30464 231.57 876756.34
132 30592 231.51 880832.15
133 30848 231.33 888534.77
134 31104 231.82 898721.44
135 31360 231.64 906209.71
136 31616 231.54 913813.42
137 31872 231.59 921916.64
138 32000 231.60 925852.38
139 32256 232.01 935230.01
140 32512 231.59 941138.21
141 32768 231.37 947690.01
142 32896 230.96 949694.09
143 33152 231.63 959761.73
144 33536 232.14 972602.68
145 33664 231.67 974178.25
146 33920 232.13 983077.98
147 34176 232.17 990133.00
148 34560 232.46 1001547.46
149 34816 232.93 1010208.08
150 35072 233.06 1017327.45
151 35200 232.85 1019642.47
152 35328 232.16 1019839.87
153 35712 233.02 1033120.77
154 35968 233.39 1040997.63
155 36352 234.01 1053022.73
156 36608 234.44 1060982.19
157 36864 234.14 1065613.31
158 36992 234.04 1068111.43
159 37248 234.13 1074343.60
160 37632 234.50 1084676.37
161 37760 233.84 1084438.81
162 37888 233.34 1084926.16
163 38144 233.51 1091263.99
164 38400 233.66 1097440.66
165 38656 233.47 1101974.43
166 38784 233.45 1104537.64
167 39040 233.77 1111333.08
168 39296 233.83 1116894.88
169 39552 233.43 1120133.47
170 39680 233.05 1120858.39
171 39936 232.64 1123929.79
172 40192 233.44 1132809.48
173 40448 233.08 1135945.76
174 40576 233.05 1138268.49
175 40960 232.92 1144869.25
176 41088 233.28 1149000.83
177 41344 233.01 1152388.03
178 41472 232.78 1153586.56
179 41728 232.54 1157001.98
180 41856 232.47 1158918.42
181 42112 232.37 1162962.38
182 42368 232.28 1167006.58
183 42496 232.15 1168577.80
184 42752 231.80 1171157.45
185 42880 231.70 1172828.11
186 43136 231.90 1178149.03
187 43392 232.03 1183078.91
188 43776 232.34 1190992.75
189 44032 232.65 1196705.15
190 44288 232.28 1198914.76
191 44416 232.41 1201598.08
192 44800 232.64 1208811.68
193 44928 232.77 1211488.61
194 45184 232.57 1214376.02
195 45568 233.31 1224095.81
196 45824 233.02 1226367.45
197 46080 233.48 1232618.10
198 46208 233.12 1232609.63
199 46464 233.03 1235848.01
200 46592 232.94 1237227.72
201 46976 233.44 1245374.59
202 47104 232.97 1244687.87
203 47488 233.39 1252306.11
204 47616 233.28 1253463.36
205 47872 233.36 1257405.22
206 48128 233.61 1262235.18
207 48512 234.16 1270416.92
208 48768 233.53 1270374.51
209 48896 233.53 1272039.15
210 49152 233.61 1275801.88
211 49280 233.50 1276859.63
212 49664 233.80 1283466.15
213 49792 233.43 1283039.34
214 50048 233.85 1288557.19
215 50304 233.87 1291861.57
216 50688 234.02 1297398.66
217 50816 234.15 1299731.79
218 51072 233.93 1301601.03
219 51328 234.21 1306210.23
220 51584 234.12 1308755.54
221 51840 234.16 1311985.82
222 51968 233.96 1312371.07
223 52224 234.17 1316470.53
224 52480 233.83 1317542.09
225 52736 234.04 1321620.65
226 52864 233.63 1320710.56
227 53120 233.70 1323987.67
228 53376 233.82 1327482.51
229 53632 233.20 1326754.01
230 53760 233.33 1328879.16
231 53888 233.25 1329830.94
232 54144 233.14 1331949.25
233 54272 232.86 1331730.40
234 54528 232.33 1331356.25
235 54656 232.34 1332786.38
236 55040 232.46 1337457.12
237 55168 232.60 1339574.94
238 55424 232.31 1340523.79
239 55680 232.85 1346196.08
240 55936 232.41 1346223.91
241 56064 232.52 1348124.20
242 56320 232.49 1350522.36
243 56576 232.64 1353905.61
244 56832 232.63 1356337.96
245 57088 232.33 1357032.25
246 57216 232.21 1357520.85
247 57472 232.15 1359641.48
248 57600 232.14 1360752.36
249 57984 232.53 1366660.06
250 58240 232.73 1370185.18
251 58496 233.04 1374336.35
252 58752 233.07 1376847.82
253 59008 232.49 1375762.81
254 59136 232.50 1376941.91
255 59264 232.29 1376840.52
256 59520 232.31 1379202.73
257 59776 232.47 1382448.26
258 60032 232.51 1384890.91
259 60288 232.14 1384892.07
260 60416 232.30 1386957.80
261 60800 232.77 1393008.77
262 61056 232.50 1393526.48
263 61184 232.26 1393145.13
264 61312 232.13 1393475.70
265 61568 232.30 1396589.50
266 61824 232.31 1398739.33
267 62080 232.34 1400996.22
268 62336 232.07 1401443.18
269 62464 232.09 1402597.33
270 62720 232.21 1405360.99
271 62976 231.87 1405326.22
272 63232 231.72 1406409.55
273 63360 231.90 1408496.78
274 63744 232.22 1413399.95
275 63872 231.91 1412463.80
276 64128 232.26 1416557.75
277 64384 232.14 1417748.57
278 64640 232.46 1421623.16
279 64896 232.13 1421518.14
280 65152 232.45 1425360.40
281 65408 232.62 1428310.41
282 65664 232.79 1431207.24
283 65920 232.58 1431754.11
284 66176 232.55 1433421.40
285 66432 232.68 1436038.13
286 66560 232.61 1436512.32
287 66688 232.34 1435705.22
288 66944 232.37 1437696.75
289 67328 232.61 1441815.77
290 67584 232.79 1444732.09
291 67712 232.44 1443431.35
292 67968 232.70 1446762.19
293 68224 232.56 1447622.23
294 68480 232.54 1449199.34
295 68608 232.42 1449291.26
296 68864 232.56 1451845.42
297 69248 232.78 1455739.56
298 69376 232.79 1456670.08
299 69632 232.53 1456686.83
300 69760 232.31 1456136.78
301 70016 232.34 1457916.74
302 70144 232.13 1457409.65
303 70528 232.54 1462408.53
304 70784 232.40 1463161.31
305 71040 232.44 1464985.93
306 71296 232.65 1467871.69
307 71552 232.62 1469269.22
308 71808 232.90 1472629.94
309 72064 233.10 1475385.15
310 72448 233.08 1477563.06
311 72576 233.16 1478881.83
312 72832 232.85 1478403.16
313 72960 232.98 1479991.79
314 73088 232.63 1478482.65
315 73344 232.63 1479984.29
316 73600 232.35 1479697.69
317 73728 232.24 1479709.90
318 73856 232.08 1479412.80
319 73984 231.84 1478642.34
320 74240 231.73 1479367.59
321 74496 232.02 1482676.27
322 74880 232.32 1486716.36
323 75008 232.22 1486829.92
324 75264 232.02 1486918.64
325 75520 232.12 1489011.05
326 75776 232.37 1492011.50
327 76032 232.37 1493368.04
328 76288 232.47 1495428.48
329 76672 232.74 1499215.10
330 76800 232.56 1498683.95
331 77056 232.54 1499924.76
332 77312 232.79 1502890.16
333 77568 232.76 1504040.09
334 77696 232.62 1503768.78
335 77824 232.24 1501990.12
336 78080 231.89 1501049.17
337 78336 232.06 1503408.18
338 78464 231.80 1502422.81
339 78592 231.80 1503028.10
340 78848 231.85 1504650.47
341 79104 231.59 1504241.20
342 79232 231.52 1504438.14
343 79488 231.68 1506694.18
344 79744 231.73 1508281.37
345 80000 231.49 1508003.78
346 80128 231.53 1508893.63
347 80384 231.52 1510031.74
348 80640 231.28 1509726.62
349 80768 231.39 1511056.77
350 81152 231.54 1513857.71
351 81408 231.91 1517435.76
352 81664 231.60 1516647.39
353 81792 231.68 1517751.00
354 82176 231.89 1520914.77
355 82304 231.81 1520967.71
356 82560 231.80 1522046.58
357 82944 231.99 1525075.56
358 83200 231.92 1525754.40
359 83328 231.79 1525498.33
360 83584 231.85 1527001.08
361 83840 232.15 1530127.04
362 84096 232.12 1531077.05
363 84352 232.18 1532575.38
364 84608 232.36 1534887.05
365 84864 232.37 1536073.81
366 85120 232.36 1537092.48
367 85376 232.44 1538708.79
368 85632 232.53 1540445.74
369 85888 232.65 1542305.23
370 86144 232.76 1544089.41
371 86528 232.73 1545525.12
372 86656 232.82 1546626.30
373 86912 232.70 1546893.71
374 87168 232.79 1548561.23
375 87296 232.74 1548744.55
376 87680 232.75 1550387.44
377 87808 232.64 1550185.86
378 88064 232.65 1551266.43
379 88320 232.86 1553748.27
380 88576 233.01 1555741.46
381 88832 232.88 1555867.15
382 88960 232.81 1555947.50
383 89344 233.05 1559023.40
384 89600 232.90 1559050.92
385 89728 232.72 1558350.53
386 89856 232.63 1558254.33
387 89984 232.49 1557783.54
388 90240 232.47 1558647.56
389 90496 232.55 1560153.72
390 90880 232.86 1563686.26
391 91136 232.83 1564422.64
392 91264 232.72 1564174.41
393 91648 232.97 1567338.02
394 91776 232.65 1565605.57
395 91904 232.66 1566170.23
396 92160 232.34 1564930.59
397 92288 232.40 1565862.30
398 92544 232.46 1567192.84
399 92800 232.39 1567658.73
400 93056 232.30 1567942.90
401 93184 232.08 1566942.43
402 93312 232.04 1567138.96
403 93568 232.11 1568523.49
404 93952 232.18 1570361.79
405 94080 232.24 1571205.67
406 94336 232.21 1571890.85
407 94592 232.14 1572312.30
408 94720 232.05 1572147.37
409 95104 232.34 1575417.27
410 95360 232.35 1576350.24
411 95488 232.31 1576524.82
412 95872 232.50 1579150.69
413 96128 232.69 1581324.36
414 96384 232.60 1581567.19
415 96512 232.53 1581506.87
416 96768 232.47 1581943.21
417 96896 232.33 1581403.29
418 97152 232.32 1582183.55
419 97408 232.18 1582096.02
420 97664 232.45 1584750.44
421 97920 232.27 1584363.51
422 98048 232.04 1583250.75
423 98304 232.23 1585359.50
424 98560 231.96 1584342.63
425 98688 232.19 1586309.25
426 98944 232.21 1587298.48
427 99200 232.27 1588471.40
428 99456 232.29 1589460.85
429 99712 232.42 1591152.29
430 99968 232.33 1591335.10
431 100224 232.46 1592999.68
432 100480 232.38 1593238.81
433 100736 232.48 1594714.86
434 100864 232.35 1594202.89
435 101248 232.63 1597290.61
436 101504 232.61 1597942.45
437 101760 232.69 1599292.67
438 102016 232.80 1600822.95
439 102144 232.67 1600319.75
440 102656 233.13 1605019.55
441 102912 233.31 1606966.57
442 103040 233.10 1605952.67
443 103424 233.19 1607675.49
444 103552 233.04 1606986.95
445 103808 232.86 1606523.93
446 103936 232.81 1606546.01
447 104064 232.75 1606504.49
448 104448 232.88 1608484.82
449 104704 232.97 1609861.09
450 104832 232.95 1610077.05
451 105088 232.89 1610388.55
452 105216 232.77 1609932.87
453 105600 232.90 1611912.68
454 105856 232.84 1612232.95
455 105984 232.92 1613102.05
456 106368 233.04 1614987.14
457 106496 233.02 1615198.01
458 106752 232.97 1615590.18
459 107008 233.07 1617005.53
460 107264 232.87 1616268.53
461 107392 232.82 1616303.92
462 107648 232.88 1617402.90
463 107904 232.85 1617883.26
464 108032 232.72 1617333.58
465 108288 232.83 1618787.43
466 108672 233.06 1621382.39
467 108800 232.86 1620305.05
468 109056 232.81 1620629.10
469 109312 232.89 1621875.98
470 109568 233.06 1623768.22
471 109952 233.15 1625328.41
472 110080 232.95 1624327.75
473 110208 232.99 1624904.65
474 110464 233.03 1625823.90
475 110848 233.08 1627192.38
476 111104 233.27 1629166.24
477 111360 233.37 1630526.06
478 111488 233.21 1629687.05
479 111744 232.87 1627987.09
480 111872 232.97 1628995.16
481 112128 232.91 1629224.59
482 112384 233.00 1630468.93
483 112512 232.72 1628799.10
484 112768 232.95 1631101.24
485 113152 233.06 1632782.71
486 113408 233.02 1633128.59
487 113536 232.86 1632326.75
488 113664 232.87 1632673.28
489 113920 232.90 1633518.93
490 114176 232.90 1634145.00
491 114432 232.90 1634723.85
492 114688 232.87 1635139.00
493 114816 232.79 1634881.11
494 115072 232.81 1635644.69
495 115328 232.97 1637378.69
496 115584 233.00 1638179.84
497 115840 233.05 1639100.83
498 116224 233.32 1641904.97
499 116480 233.34 1642648.93
500 116864 233.45 1644320.13
501 116992 233.40 1644236.15
502 117248 233.22 1643570.02
503 117376 233.11 1643086.47
504 117504 232.98 1642418.03
505 117632 232.89 1642128.87
506 117888 232.57 1640423.36
507 118016 232.58 1640813.37
508 118272 232.47 1640615.53
509 118528 232.71 1642819.80
510 118656 232.49 1641564.43
511 118784 232.44 1641533.63
512 119168 232.71 1644245.76
513 119424 232.71 1644825.81
514 119680 232.77 1645827.30
515 119936 232.75 1646213.24
516 120192 232.73 1646636.42
517 120320 232.71 1646797.46
518 120704 232.87 1648754.45
519 120832 232.78 1648345.89
520 121216 232.97 1650501.10
521 121472 233.09 1651908.39
522 121728 233.11 1652618.27
523 121984 233.08 1652918.89
524 122240 233.15 1653991.51
525 122496 233.27 1655363.47
526 122752 233.36 1656502.07
527 123008 233.35 1656970.85
528 123264 233.25 1656834.60
529 123520 233.19 1656894.55
530 123648 233.15 1656871.22
531 123904 233.17 1657536.76
532 124160 233.25 1658617.57
533 124416 233.36 1659907.07
534 124672 233.42 1660901.59
535 124928 233.12 1659239.30
536 125056 233.12 1659512.93
537 125312 233.12 1660054.31
538 125440 233.04 1659728.61
539 125824 233.17 1661406.86
540 125952 233.21 1661963.68
541 126208 233.26 1662775.61
542 126592 233.46 1665010.68
543 126848 233.48 1665609.17
544 127104 233.58 1666843.58
545 127232 233.43 1666025.38
546 127616 233.61 1668034.11
547 127872 233.70 1669174.40
548 128128 233.67 1669472.13
549 128384 233.49 1668618.19
550 128512 233.57 1669461.17
551 128768 233.58 1670052.55
552 128896 233.42 1669099.47
553 129152 233.48 1670003.92
554 129408 233.50 1670646.41
555 129536 233.35 1669850.64
556 129920 233.54 1671886.88
557 130176 233.46 1671776.35
558 130304 233.29 1670813.93
559 130560 233.24 1670940.66
560 130688 233.28 1671468.77
561 130944 233.29 1672003.56
elapsed: 561 ops: 131072 ops/sec: 233.37 bytes/sec: 1911729.44
|
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
