hi, Jens Axboe,
On Wed, Jan 31, 2024 at 11:42:46AM -0700, Jens Axboe wrote:
> On 1/31/24 11:17 AM, Bart Van Assche wrote:
> > On 1/31/24 07:42, kernel test robot wrote:
> >> kernel test robot noticed a -72.9% regression of fio.write_iops on:
> >>
> >>
> >> commit: 574e7779cf583171acb5bf6365047bb0941b387c ("block/mq-deadline: use separate insertion lists")
> >> https://git.kernel.org/cgit/linux/kernel/git/next/linux-next.git master
> >>
> >> testcase: fio-basic
> >> test machine: 64 threads 2 sockets Intel(R) Xeon(R) Gold 6346 CPU @ 3.10GHz (Ice Lake) with 256G memory
> >> parameters:
> >>
> >> runtime: 300s
> >> disk: 1HDD
> >> fs: xfs
> >> nr_task: 100%
> >> test_size: 128G
> >> rw: write
> >> bs: 4k
> >> ioengine: io_uring
> >> direct: direct
> >> cpufreq_governor: performance
> >
> > The actual test is available in this file:
> > https://download.01.org/0day-ci/archive/20240131/202401312320.a335db14-oliver.sang@xxxxxxxxx/repro-script
> >
> > I haven't found anything in that file for disabling merging. Merging
> > requests decreases IOPS. Does this perhaps mean that this test is
> > broken?
>
> It's hard to know as nothing in this email or links include the actual
> output of the job...
I attached a dmesg and 2 outputs while running tests on 574e7779cf. not sure if
they are helpful?
>
> But if it's fio IOPS, then those are application side and don't
> necessarily correlate to drive IOPS due to merging. Eg for fio iops, if
> it does 4k sequential and we merge to 128k, then the fio perceived iops
> will be 32 times larger than the device side.
>
> I'll take a look, but seems like there might be something there. By
> inserting into the other list, the request is also not available for
> merging. And the test in question does single IOs at the time.
if you have any debug patch want us to run, please just let us know.
it will be our great pleasure!
>
> --
> Jens Axboe
>
Attachment:
dmesg.xz
Description: application/xz
2024-01-30 16:11:43 echo '[global] bs=4k ioengine=io_uring iodepth=32 size=2147483648 nr_files=1 filesize=2147483648 direct=1 runtime=300 invalidate=1 fallocate=posix io_size=2147483648 file_service_type=roundrobin random_distribution=random group_reporting pre_read=0 [task_0] rw=write directory=/fs/sdb1 numjobs=64' | fio --output-format=json -
{
"fio version" : "fio-3.33",
"timestamp" : 1706631404,
"timestamp_ms" : 1706631404540,
"time" : "Tue Jan 30 16:16:44 2024",
"global options" : {
"bs" : "4k",
"ioengine" : "io_uring",
"iodepth" : "32",
"size" : "2147483648",
"nrfiles" : "1",
"filesize" : "2147483648",
"direct" : "1",
"runtime" : "300",
"invalidate" : "1",
"fallocate" : "posix",
"io_size" : "2147483648",
"file_service_type" : "roundrobin",
"random_distribution" : "random",
"pre_read" : "0"
},
"jobs" : [
{
"jobname" : "task_0",
"groupid" : 0,
"error" : 0,
"eta" : 0,
"elapsed" : 301,
"job options" : {
"rw" : "write",
"directory" : "/fs/sdb1",
"numjobs" : "64"
},
"read" : {
"io_bytes" : 0,
"io_kbytes" : 0,
"bw_bytes" : 0,
"bw" : 0,
"iops" : 0.000000,
"runtime" : 0,
"total_ios" : 0,
"short_ios" : 0,
"drop_ios" : 0,
"slat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
},
"clat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
},
"lat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
},
"bw_min" : 0,
"bw_max" : 0,
"bw_agg" : 0.000000,
"bw_mean" : 0.000000,
"bw_dev" : 0.000000,
"bw_samples" : 0,
"iops_min" : 0,
"iops_max" : 0,
"iops_mean" : 0.000000,
"iops_stddev" : 0.000000,
"iops_samples" : 0
},
"write" : {
"io_bytes" : 7811690496,
"io_kbytes" : 7628604,
"bw_bytes" : 26013908,
"bw" : 25404,
"iops" : 6351.051820,
"runtime" : 300289,
"total_ios" : 1907151,
"short_ios" : 0,
"drop_ios" : 0,
"slat_ns" : {
"min" : 326,
"max" : 866214376,
"mean" : 9843550.857783,
"stddev" : 36978079.656282,
"N" : 1907151
},
"clat_ns" : {
"min" : 182009,
"max" : 1499097001,
"mean" : 312417107.300301,
"stddev" : 178075607.629576,
"N" : 1907151,
"percentile" : {
"1.000000" : 36438016,
"5.000000" : 64749568,
"10.000000" : 92798976,
"20.000000" : 145752064,
"30.000000" : 191889408,
"40.000000" : 238026752,
"50.000000" : 291504128,
"60.000000" : 350224384,
"70.000000" : 408944640,
"80.000000" : 471859200,
"90.000000" : 549453824,
"95.000000" : 616562688,
"99.000000" : 775946240,
"99.500000" : 859832320,
"99.900000" : 1035993088,
"99.950000" : 1115684864,
"99.990000" : 1249902592
}
},
"lat_ns" : {
"min" : 318070,
"max" : 1499097433,
"mean" : 322260658.158085,
"stddev" : 180107939.833017,
"N" : 1907151
},
"bw_min" : 2800,
"bw_max" : 80728,
"bw_agg" : 99.991312,
"bw_mean" : 25402.113478,
"bw_dev" : 183.115387,
"bw_samples" : 38394,
"iops_min" : 700,
"iops_max" : 20182,
"iops_mean" : 6350.528370,
"iops_stddev" : 45.778847,
"iops_samples" : 38394
},
"trim" : {
"io_bytes" : 0,
"io_kbytes" : 0,
"bw_bytes" : 0,
"bw" : 0,
"iops" : 0.000000,
"runtime" : 0,
"total_ios" : 0,
"short_ios" : 0,
"drop_ios" : 0,
"slat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
},
"clat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
},
"lat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
},
"bw_min" : 0,
"bw_max" : 0,
"bw_agg" : 0.000000,
"bw_mean" : 0.000000,
"bw_dev" : 0.000000,
"bw_samples" : 0,
"iops_min" : 0,
"iops_max" : 0,
"iops_mean" : 0.000000,
"iops_stddev" : 0.000000,
"iops_samples" : 0
},
"sync" : {
"total_ios" : 0,
"lat_ns" : {
"min" : 0,
"max" : 0,
"mean" : 0.000000,
"stddev" : 0.000000,
"N" : 0
}
},
"job_runtime" : 19210274,
"usr_cpu" : 0.005950,
"sys_cpu" : 0.030385,
"ctx" : 446567,
"majf" : 0,
"minf" : 883,
"iodepth_level" : {
"1" : 0.100000,
"2" : 0.100000,
"4" : 0.100000,
"8" : 0.100000,
"16" : 0.100000,
"32" : 99.895970,
">=64" : 0.000000
},
"iodepth_submit" : {
"0" : 0.000000,
"4" : 100.000000,
"8" : 0.000000,
"16" : 0.000000,
"32" : 0.000000,
"64" : 0.000000,
">=64" : 0.000000
},
"iodepth_complete" : {
"0" : 0.000000,
"4" : 99.996641,
"8" : 0.000000,
"16" : 0.000000,
"32" : 0.100000,
"64" : 0.000000,
">=64" : 0.000000
},
"latency_ns" : {
"2" : 0.000000,
"4" : 0.000000,
"10" : 0.000000,
"20" : 0.000000,
"50" : 0.000000,
"100" : 0.000000,
"250" : 0.000000,
"500" : 0.000000,
"750" : 0.000000,
"1000" : 0.000000
},
"latency_us" : {
"2" : 0.000000,
"4" : 0.000000,
"10" : 0.000000,
"20" : 0.000000,
"50" : 0.000000,
"100" : 0.000000,
"250" : 0.010000,
"500" : 0.010000,
"750" : 0.010000,
"1000" : 0.010000
},
"latency_ms" : {
"2" : 0.010000,
"4" : 0.033925,
"10" : 0.247909,
"20" : 0.190231,
"50" : 2.933171,
"100" : 7.702536,
"250" : 31.258039,
"500" : 41.803822,
"750" : 14.596065,
"1000" : 1.079935,
"2000" : 0.153056,
">=2000" : 0.000000
},
"latency_depth" : 32,
"latency_target" : 0,
"latency_percentile" : 100.000000,
"latency_window" : 0
}
],
"disk_util" : [
{
"name" : "sdb",
"read_ios" : 0,
"write_ios" : 1013029,
"read_merges" : 0,
"write_merges" : 894106,
"read_ticks" : 0,
"write_ticks" : 53607123,
"in_queue" : 53609981,
"util" : 96.247980
}
]
}
