I don't think there are here people advising to use consumer grade ssd's/nvme's. The enterprise ssd's often have more twpd, and are just stable under high constant load.
My 1,5 year old sm863a still has 099 wearlevel and 097 poweronhours, some other sm863a of 3,8 years has 099 wearlevel and 093 poweronhours.
When starting to use ceph try to keep your environment as simple as possible, and as standard as possible. You need to be aware of quite some details if you are trying to squeeze out the the last few % ceph can offer.
For instance disabling the ssd/hdd drive cache gives you better performance (I do not really notice it, but that is probably because my cluster is having low load)
Also realize from this page of Vitalif, at some point it will not become any faster. I was also thinking of putting wal/db on ssd's for the hdd pool. But I skipped it for now. The hdd pool is not fast, but I am also not complaining about it.
This is my fio[1] and a micron sata ssd drive result[2]
[1]
[global]
ioengine=libaio
#ioengine=posixaio
invalidate=1
ramp_time=30
iodepth=1
runtime=180
time_based
direct=1
filename=/dev/sdX
#filename=/mnt/cephfs/ssd/fio-bench.img
[write-4k-seq]
stonewall
bs=4k
rw=write
[randwrite-4k-seq]
stonewall
bs=4k
rw=randwrite
fsync=1
[randwrite-4k-d32-seq]
stonewall
bs=4k
rw=randwrite
iodepth=32
[read-4k-seq]
stonewall
bs=4k
rw=read
[randread-4k-seq]
stonewall
bs=4k
rw=randread
fsync=1
[randread-4k-d32-seq]
stonewall
bs=4k
rw=randread
iodepth=32
[rw-4k-seq]
stonewall
bs=4k
rw=rw
[randrw-4k-seq]
stonewall
bs=4k
rw=randrw
[randrw-4k-d4-seq]
stonewall
bs=4k
rw=randrw
iodepth=4
[write-128k-seq]
stonewall
bs=128k
rw=write
[randwrite-128k-seq]
stonewall
bs=128k
rw=randwrite
[read-128k-seq]
stonewall
bs=128k
rw=read
[randread-128k-seq]
stonewall
bs=128k
rw=randread
[rw-128k-seq]
stonewall
bs=128k
rw=rw
[randrw-128k-seq]
stonewall
bs=128k
rw=randrw
[write-1024k-seq]
stonewall
bs=1024k
rw=write
[randwrite-1024k-seq]
stonewall
bs=1024k
rw=randwrite
[read-1024k-seq]
stonewall
bs=1024k
rw=read
[randread-1024k-seq]
stonewall
bs=1024k
rw=randread
[rw-1024k-seq]
stonewall
bs=1024k
rw=rw
[randrw-1024k-seq]
stonewall
bs=1024k
rw=randrw
[write-4096k-seq]
stonewall
bs=4096k
rw=write
[write-4096k-d16-seq]
stonewall
bs=4M
rw=write
iodepth=16
[randwrite-4096k-seq]
stonewall
bs=4096k
rw=randwrite
[read-4096k-seq]
stonewall
bs=4096k
rw=read
[read-4096k-d16-seq]
stonewall
bs=4M
rw=read
iodepth=16
[randread-4096k-seq]
stonewall
bs=4096k
rw=randread
[rw-4096k-seq]
stonewall
bs=4096k
rw=rw
[randrw-4096k-seq]
stonewall
bs=4096k
rw=randrw
[2]
write-4k-seq: (groupid=0, jobs=1): err= 0: pid=982502: Sun Oct 4 16:13:28 2020
write: IOPS=15.3k, BW=59.7MiB/s (62.6MB/s)(10.5GiB/180001msec)
slat (usec): min=6, max=706, avg=12.09, stdev= 5.40
clat (nsec): min=1618, max=1154.4k, avg=50455.96, stdev=18670.50
lat (usec): min=39, max=1161, avg=62.85, stdev=21.79
clat percentiles (usec):
| 1.00th=[ 39], 5.00th=[ 40], 10.00th=[ 41], 20.00th=[ 42],
| 30.00th=[ 43], 40.00th=[ 43], 50.00th=[ 45], 60.00th=[ 48],
| 70.00th=[ 51], 80.00th=[ 54], 90.00th=[ 58], 95.00th=[ 87],
| 99.00th=[ 141], 99.50th=[ 153], 99.90th=[ 178], 99.95th=[ 188],
| 99.99th=[ 235]
bw ( KiB/s): min=37570, max=63946, per=69.50%, avg=42495.21, stdev=3251.18, samples=359
iops : min= 9392, max=15986, avg=10623.45, stdev=812.82, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=66.19%
lat (usec) : 100=30.09%, 250=3.70%, 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%
cpu : usr=9.73%, sys=29.92%, ctx=2751526, majf=0, minf=53
IO depths : 1=116.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,2751607,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randwrite-4k-seq: (groupid=1, jobs=1): err= 0: pid=983595: Sun Oct 4 16:13:28 2020
write: IOPS=14.9k, BW=58.2MiB/s (61.0MB/s)(10.2GiB/180001msec)
slat (usec): min=6, max=304, avg=10.89, stdev= 4.80
clat (nsec): min=1355, max=1258.5k, avg=49272.39, stdev=17923.95
lat (usec): min=42, max=1265, avg=60.46, stdev=20.51
clat percentiles (usec):
| 1.00th=[ 39], 5.00th=[ 40], 10.00th=[ 41], 20.00th=[ 41],
| 30.00th=[ 42], 40.00th=[ 43], 50.00th=[ 43], 60.00th=[ 46],
| 70.00th=[ 50], 80.00th=[ 53], 90.00th=[ 58], 95.00th=[ 84],
| 99.00th=[ 137], 99.50th=[ 151], 99.90th=[ 174], 99.95th=[ 184],
| 99.99th=[ 231]
bw ( KiB/s): min=37665, max=62936, per=69.49%, avg=41402.23, stdev=2934.41, samples=359
iops : min= 9416, max=15734, avg=10350.21, stdev=733.65, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=71.60%
lat (usec) : 100=25.13%, 250=3.24%, 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%
fsync/fdatasync/sync_file_range:
sync (nsec): min=71, max=98768, avg=170.36, stdev=376.47
sync percentiles (nsec):
| 1.00th=[ 102], 5.00th=[ 111], 10.00th=[ 118], 20.00th=[ 129],
| 30.00th=[ 135], 40.00th=[ 141], 50.00th=[ 147], 60.00th=[ 155],
| 70.00th=[ 167], 80.00th=[ 191], 90.00th=[ 207], 95.00th=[ 270],
| 99.00th=[ 350], 99.50th=[ 370], 99.90th=[ 506], 99.95th=[11968],
| 99.99th=[14528]
cpu : usr=12.54%, sys=29.64%, ctx=2680996, majf=0, minf=2335
IO depths : 1=233.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,2681130,0,3133981 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randwrite-4k-d32-seq: (groupid=2, jobs=1): err= 0: pid=984690: Sun Oct 4 16:13:28 2020
write: IOPS=62.0k, BW=246MiB/s (258MB/s)(43.2GiB/180001msec)
slat (usec): min=3, max=780, avg= 7.41, stdev= 3.17
clat (usec): min=229, max=3082, avg=499.32, stdev=69.67
lat (usec): min=237, max=3088, avg=506.92, stdev=69.64
clat percentiles (usec):
| 1.00th=[ 465], 5.00th=[ 482], 10.00th=[ 486], 20.00th=[ 490],
| 30.00th=[ 490], 40.00th=[ 494], 50.00th=[ 498], 60.00th=[ 498],
| 70.00th=[ 502], 80.00th=[ 506], 90.00th=[ 510], 95.00th=[ 515],
| 99.00th=[ 537], 99.50th=[ 553], 99.90th=[ 1795], 99.95th=[ 2671],
| 99.99th=[ 2704]
bw ( KiB/s): min=166034, max=253112, per=69.59%, avg=175251.29, stdev=11968.28, samples=359
iops : min=41507, max=63278, avg=43812.45, stdev=2992.12, samples=359
lat (usec) : 250=0.01%, 500=63.54%, 750=36.29%, 1000=0.05%
lat (msec) : 2=0.03%, 4=0.09%
cpu : usr=16.27%, sys=57.89%, ctx=4525456, majf=0, minf=2368
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=116.7%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued rwts: total=0,11332122,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=32
read-4k-seq: (groupid=3, jobs=1): err= 0: pid=985779: Sun Oct 4 16:13:28 2020
read: IOPS=15.6k, BW=60.7MiB/s (63.7MB/s)(10.7GiB/180001msec)
slat (usec): min=6, max=743, avg=11.63, stdev= 5.39
clat (nsec): min=1443, max=1204.6k, avg=49997.03, stdev=18517.84
lat (usec): min=39, max=1221, avg=61.93, stdev=21.59
clat percentiles (usec):
| 1.00th=[ 38], 5.00th=[ 40], 10.00th=[ 41], 20.00th=[ 42],
| 30.00th=[ 42], 40.00th=[ 43], 50.00th=[ 44], 60.00th=[ 47],
| 70.00th=[ 51], 80.00th=[ 54], 90.00th=[ 58], 95.00th=[ 86],
| 99.00th=[ 141], 99.50th=[ 153], 99.90th=[ 178], 99.95th=[ 188],
| 99.99th=[ 231]
bw ( KiB/s): min=37884, max=66128, per=69.56%, avg=43270.55, stdev=3553.93, samples=359
iops : min= 9471, max=16532, avg=10817.30, stdev=888.51, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=67.44%
lat (usec) : 100=28.98%, 250=3.56%, 500=0.01%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%
cpu : usr=9.35%, sys=29.59%, ctx=2798966, majf=0, minf=41
IO depths : 1=116.6%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=2799121,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randread-4k-seq: (groupid=4, jobs=1): err= 0: pid=986879: Sun Oct 4 16:13:28 2020
read: IOPS=8847, BW=34.6MiB/s (36.2MB/s)(6221MiB/180001msec)
slat (usec): min=6, max=353, avg=15.15, stdev= 7.79
clat (nsec): min=1604, max=1321.5k, avg=93250.84, stdev=44991.23
lat (usec): min=48, max=1343, avg=108.77, stdev=48.33
clat percentiles (usec):
| 1.00th=[ 45], 5.00th=[ 47], 10.00th=[ 48], 20.00th=[ 49],
| 30.00th=[ 52], 40.00th=[ 61], 50.00th=[ 95], 60.00th=[ 112],
| 70.00th=[ 124], 80.00th=[ 129], 90.00th=[ 145], 95.00th=[ 176],
| 99.00th=[ 221], 99.50th=[ 235], 99.90th=[ 258], 99.95th=[ 273],
| 99.99th=[ 347]
bw ( KiB/s): min=22013, max=39200, per=69.59%, avg=24626.06, stdev=2057.73, samples=359
iops : min= 5503, max= 9800, avg=6156.15, stdev=514.46, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=25.18%
lat (usec) : 100=25.55%, 250=49.11%, 500=0.16%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%
cpu : usr=8.12%, sys=22.38%, ctx=1592759, majf=0, minf=2894
IO depths : 1=116.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=1592507,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randread-4k-d32-seq: (groupid=5, jobs=1): err= 0: pid=987981: Sun Oct 4 16:13:28 2020
read: IOPS=74.2k, BW=290MiB/s (304MB/s)(50.0GiB/180001msec)
slat (usec): min=2, max=833, avg= 7.85, stdev= 3.71
clat (usec): min=154, max=2678, avg=421.44, stdev=53.01
lat (usec): min=163, max=2687, avg=429.49, stdev=52.75
clat percentiles (usec):
| 1.00th=[ 310], 5.00th=[ 347], 10.00th=[ 359], 20.00th=[ 375],
| 30.00th=[ 392], 40.00th=[ 400], 50.00th=[ 416], 60.00th=[ 433],
| 70.00th=[ 453], 80.00th=[ 469], 90.00th=[ 490], 95.00th=[ 506],
| 99.00th=[ 545], 99.50th=[ 562], 99.90th=[ 611], 99.95th=[ 668],
| 99.99th=[ 832]
bw ( KiB/s): min=198498, max=298688, per=69.59%, avg=206632.68, stdev=14327.63, samples=359
iops : min=49624, max=74672, avg=51657.88, stdev=3581.94, samples=359
lat (usec) : 250=0.04%, 500=93.03%, 750=6.90%, 1000=0.02%
lat (msec) : 2=0.01%, 4=0.01%
cpu : usr=14.91%, sys=69.72%, ctx=1946390, majf=0, minf=3154
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=116.7%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
issued rwts: total=13362335,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=32
rw-4k-seq: (groupid=6, jobs=1): err= 0: pid=989105: Sun Oct 4 16:13:28 2020
read: IOPS=7715, BW=30.1MiB/s (31.6MB/s)(5425MiB/180001msec)
slat (usec): min=6, max=211, avg=11.46, stdev= 5.37
clat (nsec): min=1413, max=1369.2k, avg=50672.76, stdev=20907.54
lat (usec): min=42, max=1377, avg=62.43, stdev=23.81
clat percentiles (usec):
| 1.00th=[ 38], 5.00th=[ 40], 10.00th=[ 40], 20.00th=[ 41],
| 30.00th=[ 42], 40.00th=[ 43], 50.00th=[ 44], 60.00th=[ 46],
| 70.00th=[ 51], 80.00th=[ 55], 90.00th=[ 62], 95.00th=[ 90],
| 99.00th=[ 145], 99.50th=[ 157], 99.90th=[ 204], 99.95th=[ 265],
| 99.99th=[ 412]
bw ( KiB/s): min=15439, max=27872, per=69.34%, avg=21399.92, stdev=1382.14, samples=359
iops : min= 3859, max= 6968, avg=5349.61, stdev=345.53, samples=359
write: IOPS=7694, BW=30.1MiB/s (31.5MB/s)(5410MiB/180001msec)
slat (usec): min=6, max=278, avg=11.87, stdev= 5.57
clat (nsec): min=1569, max=1149.9k, avg=50262.42, stdev=18712.26
lat (usec): min=42, max=1159, avg=62.43, stdev=21.87
clat percentiles (usec):
| 1.00th=[ 39], 5.00th=[ 40], 10.00th=[ 41], 20.00th=[ 42],
| 30.00th=[ 43], 40.00th=[ 43], 50.00th=[ 44], 60.00th=[ 47],
| 70.00th=[ 51], 80.00th=[ 54], 90.00th=[ 59], 95.00th=[ 87],
| 99.00th=[ 141], 99.50th=[ 153], 99.90th=[ 178], 99.95th=[ 190],
| 99.99th=[ 269]
bw ( KiB/s): min=15433, max=28376, per=69.34%, avg=21341.45, stdev=1365.48, samples=359
iops : min= 3858, max= 7094, avg=5334.98, stdev=341.39, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=67.54%
lat (usec) : 100=28.69%, 250=3.72%, 500=0.03%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%
cpu : usr=9.71%, sys=28.85%, ctx=2773609, majf=0, minf=54
IO depths : 1=116.1%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=1388759,1384959,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randrw-4k-seq: (groupid=7, jobs=1): err= 0: pid=990221: Sun Oct 4 16:13:28 2020
read: IOPS=5442, BW=21.3MiB/s (22.3MB/s)(3827MiB/180001msec)
slat (usec): min=5, max=349, avg=13.77, stdev= 6.94
clat (nsec): min=1400, max=1943.1k, avg=93028.05, stdev=62509.50
lat (usec): min=47, max=1953, avg=107.14, stdev=64.35
clat percentiles (usec):
| 1.00th=[ 45], 5.00th=[ 46], 10.00th=[ 47], 20.00th=[ 49],
| 30.00th=[ 51], 40.00th=[ 57], 50.00th=[ 88], 60.00th=[ 111],
| 70.00th=[ 120], 80.00th=[ 128], 90.00th=[ 137], 95.00th=[ 169],
| 99.00th=[ 229], 99.50th=[ 293], 99.90th=[ 947], 99.95th=[ 1172],
| 99.99th=[ 1287]
bw ( KiB/s): min=11582, max=18616, per=69.41%, avg=15110.80, stdev=936.07, samples=359
iops : min= 2895, max= 4654, avg=3777.33, stdev=234.03, samples=359
write: IOPS=5445, BW=21.3MiB/s (22.3MB/s)(3829MiB/180001msec)
slat (usec): min=5, max=733, avg=14.19, stdev= 7.08
clat (nsec): min=1641, max=1086.2k, avg=54199.49, stdev=24781.05
lat (usec): min=41, max=1096, avg=68.73, stdev=28.90
clat percentiles (usec):
| 1.00th=[ 39], 5.00th=[ 40], 10.00th=[ 41], 20.00th=[ 42],
| 30.00th=[ 43], 40.00th=[ 44], 50.00th=[ 45], 60.00th=[ 49],
| 70.00th=[ 52], 80.00th=[ 56], 90.00th=[ 84], 95.00th=[ 121],
| 99.00th=[ 153], 99.50th=[ 163], 99.90th=[ 190], 99.95th=[ 206],
| 99.99th=[ 251]
bw ( KiB/s): min=11411, max=19320, per=69.42%, avg=15120.22, stdev=988.46, samples=359
iops : min= 2852, max= 4830, avg=3779.72, stdev=247.15, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=46.23%
lat (usec) : 100=25.73%, 250=27.71%, 500=0.18%, 750=0.06%, 1000=0.04%
lat (msec) : 2=0.04%
cpu : usr=9.06%, sys=24.70%, ctx=1960045, majf=0, minf=3017
IO depths : 1=114.3%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=979684,980222,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randrw-4k-d4-seq: (groupid=8, jobs=1): err= 0: pid=991322: Sun Oct 4 16:13:28 2020
read: IOPS=23.4k, BW=91.4MiB/s (95.9MB/s)(16.1GiB/180001msec)
slat (usec): min=3, max=1615, avg= 6.89, stdev= 3.25
clat (usec): min=13, max=2184, avg=102.47, stdev=96.98
lat (usec): min=42, max=2193, avg=109.52, stdev=97.01
clat percentiles (usec):
| 1.00th=[ 42], 5.00th=[ 45], 10.00th=[ 49], 20.00th=[ 56],
| 30.00th=[ 61], 40.00th=[ 67], 50.00th=[ 75], 60.00th=[ 112],
| 70.00th=[ 125], 80.00th=[ 135], 90.00th=[ 147], 95.00th=[ 159],
| 99.00th=[ 545], 99.50th=[ 832], 99.90th=[ 1270], 99.95th=[ 1319],
| 99.99th=[ 1450]
bw ( KiB/s): min=61673, max=94368, per=69.59%, avg=65155.29, stdev=4373.04, samples=359
iops : min=15418, max=23592, avg=16288.48, stdev=1093.32, samples=359
write: IOPS=23.4k, BW=91.5MiB/s (95.0MB/s)(16.1GiB/180001msec)
slat (usec): min=3, max=960, avg= 7.10, stdev= 3.24
clat (nsec): min=1625, max=1255.4k, avg=51090.07, stdev=13426.86
lat (usec): min=36, max=1264, avg=58.36, stdev=13.43
clat percentiles (usec):
| 1.00th=[ 36], 5.00th=[ 37], 10.00th=[ 38], 20.00th=[ 41],
| 30.00th=[ 44], 40.00th=[ 47], 50.00th=[ 50], 60.00th=[ 53],
| 70.00th=[ 56], 80.00th=[ 60], 90.00th=[ 64], 95.00th=[ 69],
| 99.00th=[ 96], 99.50th=[ 123], 99.90th=[ 169], 99.95th=[ 178],
| 99.99th=[ 237]
bw ( KiB/s): min=61852, max=94336, per=69.59%, avg=65220.33, stdev=4300.92, samples=359
iops : min=15463, max=23584, avg=16304.73, stdev=1075.28, samples=359
lat (usec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%, 50=31.52%
lat (usec) : 100=46.27%, 250=21.10%, 500=0.55%, 750=0.25%, 1000=0.16%
lat (msec) : 2=0.15%, 4=0.01%
cpu : usr=13.16%, sys=43.69%, ctx=4806536, majf=0, minf=2291
IO depths : 1=0.1%, 2=0.1%, 4=116.6%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=4213134,4217599,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=4
write-128k-seq: (groupid=9, jobs=1): err= 0: pid=992431: Sun Oct 4 16:13:28 2020
write: IOPS=2653, BW=332MiB/s (348MB/s)(58.3GiB/180001msec)
slat (usec): min=11, max=289, avg=35.73, stdev= 9.19
clat (usec): min=70, max=2852, avg=335.09, stdev=73.76
lat (usec): min=288, max=2907, avg=371.33, stdev=75.99
clat percentiles (usec):
| 1.00th=[ 281], 5.00th=[ 289], 10.00th=[ 289], 20.00th=[ 297],
| 30.00th=[ 306], 40.00th=[ 318], 50.00th=[ 326], 60.00th=[ 338],
| 70.00th=[ 351], 80.00th=[ 367], 90.00th=[ 392], 95.00th=[ 404],
| 99.00th=[ 433], 99.50th=[ 445], 99.90th=[ 652], 99.95th=[ 2671],
| 99.99th=[ 2737]
bw ( KiB/s): min=210691, max=360704, per=69.50%, avg=236061.37, stdev=18930.00, samples=359
iops : min= 1646, max= 2818, avg=1843.74, stdev=147.92, samples=359
lat (usec) : 100=0.01%, 250=0.01%, 500=99.85%, 750=0.06%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.07%
cpu : usr=4.68%, sys=12.77%, ctx=477833, majf=0, minf=176
IO depths : 1=116.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,477660,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randwrite-128k-seq: (groupid=10, jobs=1): err= 0: pid=993529: Sun Oct 4 16:13:28 2020
write: IOPS=2633, BW=329MiB/s (345MB/s)(57.9GiB/180001msec)
slat (usec): min=12, max=462, avg=35.52, stdev= 9.17
clat (usec): min=9, max=2866, avg=336.82, stdev=72.55
lat (usec): min=290, max=2913, avg=372.85, stdev=74.81
clat percentiles (usec):
| 1.00th=[ 281], 5.00th=[ 289], 10.00th=[ 293], 20.00th=[ 297],
| 30.00th=[ 310], 40.00th=[ 322], 50.00th=[ 330], 60.00th=[ 338],
| 70.00th=[ 355], 80.00th=[ 367], 90.00th=[ 392], 95.00th=[ 408],
| 99.00th=[ 437], 99.50th=[ 453], 99.90th=[ 676], 99.95th=[ 2671],
| 99.99th=[ 2737]
bw ( KiB/s): min=207590, max=340480, per=69.49%, avg=234242.08, stdev=17657.60, samples=359
iops : min= 1621, max= 2660, avg=1829.51, stdev=138.00, samples=359
lat (usec) : 10=0.01%, 50=0.01%, 100=0.01%, 250=0.01%, 500=99.84%
lat (usec) : 750=0.07%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.07%
cpu : usr=5.11%, sys=12.62%, ctx=474251, majf=0, minf=2007
IO depths : 1=116.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,474066,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
read-128k-seq: (groupid=11, jobs=1): err= 0: pid=994601: Sun Oct 4 16:13:28 2020
read: IOPS=2799, BW=350MiB/s (367MB/s)(61.5GiB/180001msec)
slat (usec): min=10, max=307, avg=35.97, stdev= 9.32
clat (usec): min=71, max=937, avg=315.77, stdev=37.32
lat (usec): min=276, max=980, avg=352.24, stdev=41.15
clat percentiles (usec):
| 1.00th=[ 269], 5.00th=[ 273], 10.00th=[ 277], 20.00th=[ 281],
| 30.00th=[ 285], 40.00th=[ 297], 50.00th=[ 310], 60.00th=[ 322],
| 70.00th=[ 334], 80.00th=[ 351], 90.00th=[ 371], 95.00th=[ 388],
| 99.00th=[ 412], 99.50th=[ 424], 99.90th=[ 453], 99.95th=[ 474],
| 99.99th=[ 668]
bw ( KiB/s): min=219502, max=358144, per=69.48%, avg=248962.07, stdev=18367.66, samples=359
iops : min= 1714, max= 2798, avg=1944.53, stdev=143.56, samples=359
lat (usec) : 100=0.01%, 250=0.01%, 500=99.96%, 750=0.03%, 1000=0.01%
cpu : usr=3.14%, sys=14.89%, ctx=504071, majf=0, minf=176
IO depths : 1=116.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=503894,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randread-128k-seq: (groupid=12, jobs=1): err= 0: pid=995683: Sun Oct 4 16:13:28 2020
read: IOPS=2257, BW=282MiB/s (296MB/s)(49.6GiB/180001msec)
slat (usec): min=11, max=356, avg=37.31, stdev= 9.60
clat (usec): min=9, max=1378, avg=397.59, stdev=61.96
lat (usec): min=290, max=1408, avg=435.43, stdev=65.91
clat percentiles (usec):
| 1.00th=[ 285], 5.00th=[ 293], 10.00th=[ 314], 20.00th=[ 343],
| 30.00th=[ 367], 40.00th=[ 383], 50.00th=[ 396], 60.00th=[ 412],
| 70.00th=[ 433], 80.00th=[ 453], 90.00th=[ 482], 95.00th=[ 502],
| 99.00th=[ 537], 99.50th=[ 545], 99.90th=[ 586], 99.95th=[ 635],
| 99.99th=[ 775]
bw ( KiB/s): min=177072, max=339200, per=69.67%, avg=201277.32, stdev=23501.77, samples=359
iops : min= 1383, max= 2650, avg=1572.02, stdev=183.65, samples=359
lat (usec) : 10=0.01%, 100=0.01%, 250=0.01%, 500=94.87%, 750=5.12%
lat (usec) : 1000=0.01%
lat (msec) : 2=0.01%
cpu : usr=3.55%, sys=12.82%, ctx=406427, majf=0, minf=2007
IO depths : 1=117.1%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=406282,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
rw-128k-seq: (groupid=13, jobs=1): err= 0: pid=996775: Sun Oct 4 16:13:28 2020
read: IOPS=1225, BW=153MiB/s (161MB/s)(26.9GiB/180001msec)
slat (usec): min=11, max=162, avg=36.60, stdev= 9.77
clat (usec): min=194, max=4022, avg=383.21, stdev=198.02
lat (usec): min=276, max=4058, avg=420.28, stdev=198.70
clat percentiles (usec):
| 1.00th=[ 273], 5.00th=[ 277], 10.00th=[ 281], 20.00th=[ 289],
| 30.00th=[ 306], 40.00th=[ 318], 50.00th=[ 330], 60.00th=[ 347],
| 70.00th=[ 363], 80.00th=[ 388], 90.00th=[ 457], 95.00th=[ 750],
| 99.00th=[ 1401], 99.50th=[ 1467], 99.90th=[ 1598], 99.95th=[ 1663],
| 99.99th=[ 1958]
bw ( KiB/s): min=88010, max=171776, per=69.51%, avg=109047.03, stdev=10373.59, samples=359
iops : min= 687, max= 1342, avg=851.45, stdev=81.08, samples=359
write: IOPS=1233, BW=154MiB/s (162MB/s)(27.1GiB/180001msec)
slat (usec): min=12, max=272, avg=42.84, stdev=10.90
clat (usec): min=80, max=2813, avg=338.71, stdev=64.69
lat (usec): min=290, max=2875, avg=382.08, stdev=68.02
clat percentiles (usec):
| 1.00th=[ 281], 5.00th=[ 289], 10.00th=[ 293], 20.00th=[ 297],
| 30.00th=[ 310], 40.00th=[ 322], 50.00th=[ 330], 60.00th=[ 343],
| 70.00th=[ 359], 80.00th=[ 375], 90.00th=[ 396], 95.00th=[ 412],
| 99.00th=[ 441], 99.50th=[ 453], 99.90th=[ 570], 99.95th=[ 1139],
| 99.99th=[ 2737]
bw ( KiB/s): min=92091, max=178176, per=69.52%, avg=109768.63, stdev=10452.14, samples=359
iops : min= 719, max= 1392, avg=857.09, stdev=81.70, samples=359
lat (usec) : 100=0.01%, 250=0.01%, 500=95.50%, 750=1.97%, 1000=0.86%
lat (msec) : 2=1.64%, 4=0.03%, 10=0.01%
cpu : usr=4.44%, sys=12.77%, ctx=442822, majf=0, minf=177
IO depths : 1=116.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=220617,222044,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randrw-128k-seq: (groupid=14, jobs=1): err= 0: pid=997891: Sun Oct 4 16:13:28 2020
read: IOPS=1103, BW=138MiB/s (145MB/s)(24.3GiB/180001msec)
slat (usec): min=14, max=307, avg=34.01, stdev= 8.45
clat (usec): min=143, max=2430, avg=478.15, stdev=275.82
lat (usec): min=289, max=2465, avg=512.73, stdev=276.22
clat percentiles (usec):
| 1.00th=[ 285], 5.00th=[ 297], 10.00th=[ 318], 20.00th=[ 351],
| 30.00th=[ 367], 40.00th=[ 379], 50.00th=[ 396], 60.00th=[ 420],
| 70.00th=[ 445], 80.00th=[ 465], 90.00th=[ 668], 95.00th=[ 1172],
| 99.00th=[ 1778], 99.50th=[ 1876], 99.90th=[ 1926], 99.95th=[ 1958],
| 99.99th=[ 2073]
bw ( KiB/s): min=84875, max=162008, per=69.50%, avg=98207.75, stdev=9869.15, samples=359
iops : min= 663, max= 1265, avg=766.77, stdev=77.16, samples=359
write: IOPS=1104, BW=138MiB/s (145MB/s)(24.3GiB/180001msec)
slat (usec): min=14, max=1082, avg=34.82, stdev= 9.08
clat (usec): min=158, max=2761, avg=342.08, stdev=37.20
lat (usec): min=288, max=2802, avg=377.50, stdev=41.56
clat percentiles (usec):
| 1.00th=[ 277], 5.00th=[ 285], 10.00th=[ 289], 20.00th=[ 306],
| 30.00th=[ 322], 40.00th=[ 338], 50.00th=[ 351], 60.00th=[ 359],
| 70.00th=[ 367], 80.00th=[ 371], 90.00th=[ 379], 95.00th=[ 388],
| 99.00th=[ 404], 99.50th=[ 412], 99.90th=[ 457], 99.95th=[ 529],
| 99.99th=[ 906]
bw ( KiB/s): min=83096, max=163328, per=69.51%, avg=98280.08, stdev=10437.64, samples=359
iops : min= 649, max= 1276, avg=767.31, stdev=81.59, samples=359
lat (usec) : 250=0.01%, 500=93.14%, 750=2.31%, 1000=1.25%
lat (msec) : 2=3.29%, 4=0.01%
cpu : usr=4.05%, sys=11.57%, ctx=397728, majf=0, minf=2512
IO depths : 1=117.9%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=198703,198824,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
write-1024k-seq: (groupid=15, jobs=1): err= 0: pid=999009: Sun Oct 4 16:13:28 2020
write: IOPS=437, BW=437MiB/s (459MB/s)(76.9GiB/180002msec)
slat (usec): min=65, max=628, avg=129.60, stdev=16.83
clat (usec): min=1867, max=11607, avg=2149.22, stdev=325.21
lat (usec): min=2071, max=11752, avg=2279.58, stdev=326.18
clat percentiles (usec):
| 1.00th=[ 2040], 5.00th=[ 2057], 10.00th=[ 2073], 20.00th=[ 2089],
| 30.00th=[ 2114], 40.00th=[ 2114], 50.00th=[ 2114], 60.00th=[ 2147],
| 70.00th=[ 2147], 80.00th=[ 2180], 90.00th=[ 2180], 95.00th=[ 2180],
| 99.00th=[ 2212], 99.50th=[ 2376], 99.90th=[ 9241], 99.95th=[ 9372],
| 99.99th=[ 9634]
bw ( KiB/s): min=291002, max=458752, per=69.40%, avg=310855.17, stdev=20535.52, samples=359
iops : min= 284, max= 448, avg=303.07, stdev=20.12, samples=359
lat (msec) : 2=0.04%, 4=99.58%, 10=0.38%, 20=0.01%
cpu : usr=1.94%, sys=5.51%, ctx=78803, majf=0, minf=805
IO depths : 1=117.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,78731,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randwrite-1024k-seq: (groupid=16, jobs=1): err= 0: pid=1000123: Sun Oct 4 16:13:28 2020
write: IOPS=437, BW=437MiB/s (459MB/s)(76.9GiB/180001msec)
slat (usec): min=64, max=583, avg=127.11, stdev=16.40
clat (usec): min=1836, max=11647, avg=2149.07, stdev=325.05
lat (usec): min=2085, max=11778, avg=2276.95, stdev=325.87
clat percentiles (usec):
| 1.00th=[ 2024], 5.00th=[ 2057], 10.00th=[ 2073], 20.00th=[ 2089],
| 30.00th=[ 2114], 40.00th=[ 2114], 50.00th=[ 2114], 60.00th=[ 2147],
| 70.00th=[ 2147], 80.00th=[ 2180], 90.00th=[ 2180], 95.00th=[ 2212],
| 99.00th=[ 2245], 99.50th=[ 2376], 99.90th=[ 9241], 99.95th=[ 9372],
| 99.99th=[ 9634]
bw ( KiB/s): min=290224, max=450560, per=69.39%, avg=310862.29, stdev=19150.82, samples=359
iops : min= 283, max= 440, avg=303.09, stdev=18.75, samples=359
lat (msec) : 2=0.03%, 4=99.58%, 10=0.39%, 20=0.01%
cpu : usr=2.09%, sys=5.34%, ctx=78810, majf=0, minf=1636
IO depths : 1=116.6%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,78745,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
read-1024k-seq: (groupid=17, jobs=1): err= 0: pid=1001203: Sun Oct 4 16:13:28 2020
read: IOPS=451, BW=452MiB/s (474MB/s)(79.5GiB/180002msec)
slat (usec): min=69, max=2081, avg=127.48, stdev=17.12
clat (usec): min=1713, max=2603, avg=2077.74, stdev=34.58
lat (usec): min=2038, max=4139, avg=2205.94, stdev=41.42
clat percentiles (usec):
| 1.00th=[ 1991], 5.00th=[ 2024], 10.00th=[ 2040], 20.00th=[ 2057],
| 30.00th=[ 2057], 40.00th=[ 2073], 50.00th=[ 2073], 60.00th=[ 2089],
| 70.00th=[ 2089], 80.00th=[ 2114], 90.00th=[ 2114], 95.00th=[ 2147],
| 99.00th=[ 2147], 99.50th=[ 2147], 99.90th=[ 2245], 99.95th=[ 2343],
| 99.99th=[ 2442]
bw ( KiB/s): min=303508, max=460800, per=69.39%, avg=321175.83, stdev=19114.77, samples=359
iops : min= 296, max= 450, avg=313.14, stdev=18.71, samples=359
lat (msec) : 2=1.26%, 4=98.74%
cpu : usr=0.66%, sys=6.93%, ctx=81434, majf=0, minf=1071
IO depths : 1=116.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=81357,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randread-1024k-seq: (groupid=18, jobs=1): err= 0: pid=1002287: Sun Oct 4 16:13:28 2020
read: IOPS=453, BW=454MiB/s (476MB/s)(79.8GiB/180001msec)
slat (usec): min=53, max=382, avg=125.95, stdev=16.53
clat (usec): min=1722, max=3009, avg=2068.38, stdev=87.38
lat (usec): min=1952, max=3129, avg=2195.06, stdev=90.60
clat percentiles (usec):
| 1.00th=[ 1926], 5.00th=[ 1942], 10.00th=[ 1958], 20.00th=[ 1991],
| 30.00th=[ 2008], 40.00th=[ 2024], 50.00th=[ 2057], 60.00th=[ 2089],
| 70.00th=[ 2114], 80.00th=[ 2147], 90.00th=[ 2180], 95.00th=[ 2212],
| 99.00th=[ 2245], 99.50th=[ 2278], 99.90th=[ 2409], 99.95th=[ 2442],
| 99.99th=[ 2638]
bw ( KiB/s): min=305283, max=464896, per=69.48%, avg=322945.88, stdev=20266.62, samples=359
iops : min= 298, max= 454, avg=314.90, stdev=19.85, samples=359
lat (msec) : 2=25.89%, 4=74.11%
cpu : usr=0.69%, sys=6.91%, ctx=81761, majf=0, minf=1310
IO depths : 1=116.6%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=81707,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
rw-1024k-seq: (groupid=19, jobs=1): err= 0: pid=1003358: Sun Oct 4 16:13:28 2020
read: IOPS=216, BW=217MiB/s (227MB/s)(38.1GiB/180002msec)
slat (usec): min=67, max=391, avg=122.41, stdev=15.56
clat (usec): min=1752, max=9295, avg=2238.79, stdev=434.91
lat (usec): min=2031, max=9413, avg=2361.93, stdev=435.60
clat percentiles (usec):
| 1.00th=[ 1975], 5.00th=[ 2008], 10.00th=[ 2024], 20.00th=[ 2040],
| 30.00th=[ 2057], 40.00th=[ 2057], 50.00th=[ 2073], 60.00th=[ 2073],
| 70.00th=[ 2089], 80.00th=[ 2147], 90.00th=[ 2802], 95.00th=[ 3490],
| 99.00th=[ 3589], 99.50th=[ 3720], 99.90th=[ 5080], 99.95th=[ 5604],
| 99.99th=[ 6194]
bw ( KiB/s): min=123272, max=239616, per=69.39%, avg=153880.61, stdev=15092.63, samples=359
iops : min= 120, max= 234, avg=149.77, stdev=14.79, samples=359
write: IOPS=215, BW=216MiB/s (226MB/s)(37.9GiB/180002msec)
slat (usec): min=63, max=568, avg=133.19, stdev=30.31
clat (usec): min=1912, max=10011, avg=2118.32, stdev=262.85
lat (usec): min=2069, max=10128, avg=2252.28, stdev=266.60
clat percentiles (usec):
| 1.00th=[ 2024], 5.00th=[ 2040], 10.00th=[ 2057], 20.00th=[ 2073],
| 30.00th=[ 2089], 40.00th=[ 2089], 50.00th=[ 2114], 60.00th=[ 2114],
| 70.00th=[ 2114], 80.00th=[ 2114], 90.00th=[ 2147], 95.00th=[ 2180],
| 99.00th=[ 2212], 99.50th=[ 2212], 99.90th=[ 6325], 99.95th=[ 9503],
| 99.99th=[10028]
bw ( KiB/s): min=119005, max=239616, per=69.36%, avg=153165.15, stdev=14886.51, samples=359
iops : min= 116, max= 234, avg=149.08, stdev=14.57, samples=359
lat (msec) : 2=1.74%, 4=97.90%, 10=0.36%, 20=0.01%
cpu : usr=1.44%, sys=5.69%, ctx=77885, majf=0, minf=813
IO depths : 1=117.1%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=38984,38816,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randrw-1024k-seq: (groupid=20, jobs=1): err= 0: pid=1004446: Sun Oct 4 16:13:28 2020
read: IOPS=207, BW=208MiB/s (218MB/s)(36.5GiB/180001msec)
slat (usec): min=69, max=451, avg=122.77, stdev=14.66
clat (usec): min=1870, max=16450, avg=2395.82, stdev=602.16
lat (usec): min=1967, max=16570, avg=2519.32, stdev=602.23
clat percentiles (usec):
| 1.00th=[ 1926], 5.00th=[ 1958], 10.00th=[ 1991], 20.00th=[ 2040],
| 30.00th=[ 2114], 40.00th=[ 2147], 50.00th=[ 2147], 60.00th=[ 2180],
| 70.00th=[ 2212], 80.00th=[ 2835], 90.00th=[ 3261], 95.00th=[ 3621],
| 99.00th=[ 4015], 99.50th=[ 4146], 99.90th=[ 7701], 99.95th=[10683],
| 99.99th=[13566]
bw ( KiB/s): min=114405, max=227328, per=69.39%, avg=147479.60, stdev=13987.90, samples=359
iops : min= 111, max= 222, avg=143.55, stdev=13.70, samples=359
write: IOPS=209, BW=210MiB/s (220MB/s)(36.9GiB/180001msec)
slat (usec): min=63, max=638, avg=133.61, stdev=29.02
clat (usec): min=1797, max=17878, avg=2122.39, stdev=355.97
lat (usec): min=2071, max=18006, avg=2256.74, stdev=358.46
clat percentiles (usec):
| 1.00th=[ 2024], 5.00th=[ 2057], 10.00th=[ 2073], 20.00th=[ 2089],
| 30.00th=[ 2089], 40.00th=[ 2089], 50.00th=[ 2114], 60.00th=[ 2114],
| 70.00th=[ 2114], 80.00th=[ 2114], 90.00th=[ 2147], 95.00th=[ 2180],
| 99.00th=[ 2212], 99.50th=[ 2245], 99.90th=[ 7832], 99.95th=[10814],
| 99.99th=[15139]
bw ( KiB/s): min=118934, max=231424, per=69.40%, avg=149183.38, stdev=13454.02, samples=359
iops : min= 116, max= 226, avg=145.21, stdev=13.16, samples=359
lat (msec) : 2=6.71%, 4=92.64%, 10=0.60%, 20=0.06%
cpu : usr=1.45%, sys=5.51%, ctx=75264, majf=0, minf=986
IO depths : 1=117.1%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=37358,37786,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
write-4096k-seq: (groupid=21, jobs=1): err= 0: pid=1005523: Sun Oct 4 16:13:28 2020
write: IOPS=117, BW=469MiB/s (492MB/s)(82.4GiB/180003msec)
slat (usec): min=378, max=1606, avg=658.55, stdev=54.99
clat (usec): min=7409, max=38933, avg=7867.03, stdev=744.97
lat (usec): min=8122, max=39632, avg=8526.50, stdev=745.80
clat percentiles (usec):
| 1.00th=[ 7635], 5.00th=[ 7701], 10.00th=[ 7701], 20.00th=[ 7767],
| 30.00th=[ 7767], 40.00th=[ 7767], 50.00th=[ 7767], 60.00th=[ 7832],
| 70.00th=[ 7832], 80.00th=[ 7832], 90.00th=[ 7898], 95.00th=[ 7898],
| 99.00th=[14484], 99.50th=[14746], 99.90th=[15008], 99.95th=[15401],
| 99.99th=[15664]
bw ( KiB/s): min=312128, max=491520, per=69.48%, avg=333520.25, stdev=21969.60, samples=359
iops : min= 76, max= 120, avg=80.94, stdev= 5.42, samples=359
lat (msec) : 10=98.96%, 20=1.04%, 50=0.01%
cpu : usr=1.60%, sys=6.65%, ctx=21156, majf=0, minf=2632
IO depths : 1=116.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,21094,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
write-4096k-d16-seq: (groupid=22, jobs=1): err= 0: pid=1006632: Sun Oct 4 16:13:28 2020
write: IOPS=122, BW=491MiB/s (515MB/s)(86.4GiB/180122msec)
slat (usec): min=565, max=1497, avg=890.51, stdev=123.72
clat (msec): min=116, max=241, avg=129.45, stdev= 3.34
lat (msec): min=117, max=242, avg=130.34, stdev= 3.34
clat percentiles (msec):
| 1.00th=[ 128], 5.00th=[ 128], 10.00th=[ 128], 20.00th=[ 128],
| 30.00th=[ 129], 40.00th=[ 129], 50.00th=[ 129], 60.00th=[ 129],
| 70.00th=[ 129], 80.00th=[ 129], 90.00th=[ 136], 95.00th=[ 136],
| 99.00th=[ 138], 99.50th=[ 138], 99.90th=[ 140], 99.95th=[ 153],
| 99.99th=[ 226]
bw ( KiB/s): min=326572, max=516096, per=69.39%, avg=348998.98, stdev=21473.05, samples=359
iops : min= 79, max= 126, avg=84.71, stdev= 5.29, samples=359
lat (msec) : 250=100.07%
cpu : usr=3.20%, sys=8.16%, ctx=22255, majf=0, minf=143
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=116.7%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,22101,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
randwrite-4096k-seq: (groupid=23, jobs=1): err= 0: pid=1007743: Sun Oct 4 16:13:28 2020
write: IOPS=117, BW=470MiB/s (493MB/s)(82.7GiB/180006msec)
slat (usec): min=305, max=5785, avg=644.75, stdev=86.33
clat (usec): min=7427, max=18792, avg=7850.14, stdev=721.33
lat (usec): min=8071, max=19478, avg=8495.90, stdev=724.12
clat percentiles (usec):
| 1.00th=[ 7635], 5.00th=[ 7635], 10.00th=[ 7701], 20.00th=[ 7701],
| 30.00th=[ 7767], 40.00th=[ 7767], 50.00th=[ 7767], 60.00th=[ 7767],
| 70.00th=[ 7832], 80.00th=[ 7832], 90.00th=[ 7898], 95.00th=[ 7898],
| 99.00th=[14484], 99.50th=[14746], 99.90th=[15008], 99.95th=[15401],
| 99.99th=[15664]
bw ( KiB/s): min=315929, max=491520, per=69.48%, avg=334670.64, stdev=22163.07, samples=359
iops : min= 77, max= 120, avg=81.22, stdev= 5.46, samples=359
lat (msec) : 10=98.96%, 20=1.04%
cpu : usr=1.53%, sys=6.57%, ctx=21261, majf=0, minf=3241
IO depths : 1=116.8%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,21167,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
read-4096k-seq: (groupid=24, jobs=1): err= 0: pid=1008851: Sun Oct 4 16:13:28 2020
read: IOPS=123, BW=494MiB/s (518MB/s)(86.8GiB/180006msec)
slat (usec): min=371, max=917, avg=596.02, stdev=56.63
clat (usec): min=7140, max=8388, avg=7499.97, stdev=54.18
lat (usec): min=7904, max=8968, avg=8096.87, stdev=43.69
clat percentiles (usec):
| 1.00th=[ 7373], 5.00th=[ 7439], 10.00th=[ 7439], 20.00th=[ 7439],
| 30.00th=[ 7504], 40.00th=[ 7504], 50.00th=[ 7504], 60.00th=[ 7504],
| 70.00th=[ 7504], 80.00th=[ 7570], 90.00th=[ 7570], 95.00th=[ 7570],
| 99.00th=[ 7635], 99.50th=[ 7701], 99.90th=[ 7701], 99.95th=[ 7767],
| 99.99th=[ 7898]
bw ( KiB/s): min=333586, max=508582, per=69.39%, avg=350761.62, stdev=21307.12, samples=359
iops : min= 81, max= 124, avg=85.15, stdev= 5.27, samples=359
lat (msec) : 10=100.00%
cpu : usr=0.14%, sys=7.75%, ctx=22270, majf=0, minf=1579
IO depths : 1=116.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=22213,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
read-4096k-d16-seq: (groupid=25, jobs=1): err= 0: pid=1009934: Sun Oct 4 16:13:28 2020
read: IOPS=128, BW=512MiB/s (537MB/s)(90.1GiB/180121msec)
slat (usec): min=444, max=2334, avg=779.85, stdev=113.42
clat (msec): min=122, max=233, avg=124.14, stdev= 1.65
lat (msec): min=124, max=234, avg=124.92, stdev= 1.64
clat percentiles (msec):
| 1.00th=[ 125], 5.00th=[ 125], 10.00th=[ 125], 20.00th=[ 125],
| 30.00th=[ 125], 40.00th=[ 125], 50.00th=[ 125], 60.00th=[ 125],
| 70.00th=[ 125], 80.00th=[ 125], 90.00th=[ 125], 95.00th=[ 125],
| 99.00th=[ 125], 99.50th=[ 125], 99.90th=[ 125], 99.95th=[ 148],
| 99.99th=[ 218]
bw ( KiB/s): min=345445, max=524987, per=69.48%, avg=364622.82, stdev=23558.65, samples=359
iops : min= 84, max= 128, avg=88.55, stdev= 5.83, samples=359
lat (msec) : 250=100.07%
cpu : usr=0.19%, sys=10.42%, ctx=23247, majf=0, minf=188
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=116.7%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.1%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=23061,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=16
randread-4096k-seq: (groupid=26, jobs=1): err= 0: pid=1011030: Sun Oct 4 16:13:28 2020
read: IOPS=125, BW=503MiB/s (528MB/s)(88.5GiB/180006msec)
slat (usec): min=369, max=972, avg=598.35, stdev=50.76
clat (usec): min=6816, max=8496, avg=7341.04, stdev=212.46
lat (usec): min=7536, max=9099, avg=7940.29, stdev=211.99
clat percentiles (usec):
| 1.00th=[ 7046], 5.00th=[ 7111], 10.00th=[ 7111], 20.00th=[ 7177],
| 30.00th=[ 7177], 40.00th=[ 7242], 50.00th=[ 7242], 60.00th=[ 7308],
| 70.00th=[ 7439], 80.00th=[ 7635], 90.00th=[ 7701], 95.00th=[ 7701],
| 99.00th=[ 7767], 99.50th=[ 7832], 99.90th=[ 7898], 99.95th=[ 7963],
| 99.99th=[ 8160]
bw ( KiB/s): min=339054, max=516096, per=69.39%, avg=357600.02, stdev=21851.65, samples=359
iops : min= 82, max= 126, avg=86.80, stdev= 5.39, samples=359
lat (msec) : 10=100.00%
cpu : usr=0.21%, sys=7.88%, ctx=22735, majf=0, minf=2154
IO depths : 1=116.7%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=22647,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
rw-4096k-seq: (groupid=27, jobs=1): err= 0: pid=1012104: Sun Oct 4 16:13:28 2020
read: IOPS=57, BW=230MiB/s (241MB/s)(40.4GiB/180005msec)
slat (usec): min=318, max=908, avg=591.91, stdev=57.56
clat (usec): min=7195, max=17434, avg=8034.82, stdev=811.74
lat (usec): min=7926, max=18037, avg=8627.64, stdev=810.57
clat percentiles (usec):
| 1.00th=[ 7373], 5.00th=[ 7439], 10.00th=[ 7439], 20.00th=[ 7504],
| 30.00th=[ 7504], 40.00th=[ 7504], 50.00th=[ 7635], 60.00th=[ 8094],
| 70.00th=[ 8225], 80.00th=[ 8848], 90.00th=[ 8848], 95.00th=[ 8979],
| 99.00th=[ 9503], 99.50th=[13304], 99.90th=[16057], 99.95th=[16581],
| 99.99th=[17433]
bw ( KiB/s): min=109372, max=289837, per=69.47%, avg=163479.04, stdev=25164.90, samples=359
iops : min= 26, max= 70, avg=39.42, stdev= 6.17, samples=359
write: IOPS=58, BW=234MiB/s (245MB/s)(41.1GiB/180005msec)
slat (usec): min=333, max=1670, avg=761.48, stdev=99.40
clat (usec): min=7449, max=18659, avg=7861.72, stdev=631.41
lat (usec): min=8115, max=19499, avg=8624.10, stdev=636.51
clat percentiles (usec):
| 1.00th=[ 7635], 5.00th=[ 7701], 10.00th=[ 7701], 20.00th=[ 7767],
| 30.00th=[ 7767], 40.00th=[ 7767], 50.00th=[ 7832], 60.00th=[ 7832],
| 70.00th=[ 7832], 80.00th=[ 7832], 90.00th=[ 7898], 95.00th=[ 7898],
| 99.00th=[ 9503], 99.50th=[14746], 99.90th=[15139], 99.95th=[15139],
| 99.99th=[15533]
bw ( KiB/s): min=106608, max=270336, per=69.50%, avg=166212.72, stdev=23683.80, samples=359
iops : min= 26, max= 66, avg=40.06, stdev= 5.80, samples=359
lat (msec) : 10=99.16%, 20=0.84%
cpu : usr=1.46%, sys=6.90%, ctx=20939, majf=0, minf=1575
IO depths : 1=116.5%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=10342,10510,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
randrw-4096k-seq: (groupid=28, jobs=1): err= 0: pid=1013179: Sun Oct 4 16:13:28 2020
read: IOPS=58, BW=232MiB/s (243MB/s)(40.8GiB/180009msec)
slat (usec): min=332, max=902, avg=594.66, stdev=53.16
clat (usec): min=6892, max=20966, avg=8047.52, stdev=769.33
lat (usec): min=7557, max=21625, avg=8643.06, stdev=769.26
clat percentiles (usec):
| 1.00th=[ 7111], 5.00th=[ 7177], 10.00th=[ 7242], 20.00th=[ 7504],
| 30.00th=[ 7635], 40.00th=[ 7701], 50.00th=[ 7767], 60.00th=[ 8029],
| 70.00th=[ 8356], 80.00th=[ 8586], 90.00th=[ 9110], 95.00th=[ 9372],
| 99.00th=[ 9896], 99.50th=[10552], 99.90th=[13698], 99.95th=[15533],
| 99.99th=[17171]
bw ( KiB/s): min=155648, max=327680, per=99.99%, avg=237583.41, stdev=32656.70, samples=360
iops : min= 38, max= 80, avg=57.95, stdev= 7.98, samples=360
write: IOPS=57, BW=230MiB/s (241MB/s)(40.5GiB/180009msec)
slat (usec): min=347, max=1804, avg=765.66, stdev=96.86
clat (usec): min=7462, max=25600, avg=7882.23, stdev=744.83
lat (usec): min=8106, max=26335, avg=8648.82, stdev=750.75
clat percentiles (usec):
| 1.00th=[ 7635], 5.00th=[ 7701], 10.00th=[ 7701], 20.00th=[ 7767],
| 30.00th=[ 7767], 40.00th=[ 7767], 50.00th=[ 7832], 60.00th=[ 7832],
| 70.00th=[ 7832], 80.00th=[ 7832], 90.00th=[ 7898], 95.00th=[ 7898],
| 99.00th=[11731], 99.50th=[14222], 99.90th=[16909], 99.95th=[17433],
| 99.99th=[24773]
bw ( KiB/s): min=155648, max=311296, per=99.98%, avg=235649.01, stdev=30116.70, samples=360
iops : min= 38, max= 76, avg=57.48, stdev= 7.35, samples=360
lat (msec) : 10=99.03%, 20=0.95%, 50=0.02%
cpu : usr=1.51%, sys=6.88%, ctx=20889, majf=0, minf=2159
IO depths : 1=117.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=10442,10358,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=1
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
