Re: [linux-next:master] [block/mq] 574e7779cf: fio.write_iops -72.9% regression

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



hi, Jens Axboe,

On Wed, Jan 31, 2024 at 11:42:46AM -0700, Jens Axboe wrote:
> On 1/31/24 11:17 AM, Bart Van Assche wrote:
> > On 1/31/24 07:42, kernel test robot wrote:
> >> kernel test robot noticed a -72.9% regression of fio.write_iops on:
> >>
> >>
> >> commit: 574e7779cf583171acb5bf6365047bb0941b387c ("block/mq-deadline: use separate insertion lists")
> >> https://git.kernel.org/cgit/linux/kernel/git/next/linux-next.git master
> >>
> >> testcase: fio-basic
> >> test machine: 64 threads 2 sockets Intel(R) Xeon(R) Gold 6346 CPU @ 3.10GHz (Ice Lake) with 256G memory
> >> parameters:
> >>
> >>     runtime: 300s
> >>     disk: 1HDD
> >>     fs: xfs
> >>     nr_task: 100%
> >>     test_size: 128G
> >>     rw: write
> >>     bs: 4k
> >>     ioengine: io_uring
> >>     direct: direct
> >>     cpufreq_governor: performance
> > 
> > The actual test is available in this file:
> > https://download.01.org/0day-ci/archive/20240131/202401312320.a335db14-oliver.sang@xxxxxxxxx/repro-script
> > 
> > I haven't found anything in that file for disabling merging. Merging
> > requests decreases IOPS. Does this perhaps mean that this test is
> > broken?
> 
> It's hard to know as nothing in this email or links include the actual
> output of the job...

I attached a dmesg and 2 outputs while running tests on 574e7779cf. not sure if
they are helpful?

> 
> But if it's fio IOPS, then those are application side and don't
> necessarily correlate to drive IOPS due to merging. Eg for fio iops, if
> it does 4k sequential and we merge to 128k, then the fio perceived iops
> will be 32 times larger than the device side.
> 
> I'll take a look, but seems like there might be something there. By
> inserting into the other list, the request is also not available for
> merging. And the test in question does single IOs at the time.

if you have any debug patch want us to run, please just let us know.
it will be our great pleasure!

> 
> -- 
> Jens Axboe
> 

Attachment: dmesg.xz
Description: application/xz

2024-01-30 16:11:43 echo '[global]
bs=4k
ioengine=io_uring
iodepth=32
size=2147483648
nr_files=1
filesize=2147483648
direct=1
runtime=300
invalidate=1
fallocate=posix
io_size=2147483648
file_service_type=roundrobin
random_distribution=random
group_reporting
pre_read=0

[task_0]
rw=write
directory=/fs/sdb1
numjobs=64' | fio --output-format=json -
{
  "fio version" : "fio-3.33",
  "timestamp" : 1706631404,
  "timestamp_ms" : 1706631404540,
  "time" : "Tue Jan 30 16:16:44 2024",
  "global options" : {
    "bs" : "4k",
    "ioengine" : "io_uring",
    "iodepth" : "32",
    "size" : "2147483648",
    "nrfiles" : "1",
    "filesize" : "2147483648",
    "direct" : "1",
    "runtime" : "300",
    "invalidate" : "1",
    "fallocate" : "posix",
    "io_size" : "2147483648",
    "file_service_type" : "roundrobin",
    "random_distribution" : "random",
    "pre_read" : "0"
  },
  "jobs" : [
    {
      "jobname" : "task_0",
      "groupid" : 0,
      "error" : 0,
      "eta" : 0,
      "elapsed" : 301,
      "job options" : {
        "rw" : "write",
        "directory" : "/fs/sdb1",
        "numjobs" : "64"
      },
      "read" : {
        "io_bytes" : 0,
        "io_kbytes" : 0,
        "bw_bytes" : 0,
        "bw" : 0,
        "iops" : 0.000000,
        "runtime" : 0,
        "total_ios" : 0,
        "short_ios" : 0,
        "drop_ios" : 0,
        "slat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "clat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "lat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "bw_min" : 0,
        "bw_max" : 0,
        "bw_agg" : 0.000000,
        "bw_mean" : 0.000000,
        "bw_dev" : 0.000000,
        "bw_samples" : 0,
        "iops_min" : 0,
        "iops_max" : 0,
        "iops_mean" : 0.000000,
        "iops_stddev" : 0.000000,
        "iops_samples" : 0
      },
      "write" : {
        "io_bytes" : 7811690496,
        "io_kbytes" : 7628604,
        "bw_bytes" : 26013908,
        "bw" : 25404,
        "iops" : 6351.051820,
        "runtime" : 300289,
        "total_ios" : 1907151,
        "short_ios" : 0,
        "drop_ios" : 0,
        "slat_ns" : {
          "min" : 326,
          "max" : 866214376,
          "mean" : 9843550.857783,
          "stddev" : 36978079.656282,
          "N" : 1907151
        },
        "clat_ns" : {
          "min" : 182009,
          "max" : 1499097001,
          "mean" : 312417107.300301,
          "stddev" : 178075607.629576,
          "N" : 1907151,
          "percentile" : {
            "1.000000" : 36438016,
            "5.000000" : 64749568,
            "10.000000" : 92798976,
            "20.000000" : 145752064,
            "30.000000" : 191889408,
            "40.000000" : 238026752,
            "50.000000" : 291504128,
            "60.000000" : 350224384,
            "70.000000" : 408944640,
            "80.000000" : 471859200,
            "90.000000" : 549453824,
            "95.000000" : 616562688,
            "99.000000" : 775946240,
            "99.500000" : 859832320,
            "99.900000" : 1035993088,
            "99.950000" : 1115684864,
            "99.990000" : 1249902592
          }
        },
        "lat_ns" : {
          "min" : 318070,
          "max" : 1499097433,
          "mean" : 322260658.158085,
          "stddev" : 180107939.833017,
          "N" : 1907151
        },
        "bw_min" : 2800,
        "bw_max" : 80728,
        "bw_agg" : 99.991312,
        "bw_mean" : 25402.113478,
        "bw_dev" : 183.115387,
        "bw_samples" : 38394,
        "iops_min" : 700,
        "iops_max" : 20182,
        "iops_mean" : 6350.528370,
        "iops_stddev" : 45.778847,
        "iops_samples" : 38394
      },
      "trim" : {
        "io_bytes" : 0,
        "io_kbytes" : 0,
        "bw_bytes" : 0,
        "bw" : 0,
        "iops" : 0.000000,
        "runtime" : 0,
        "total_ios" : 0,
        "short_ios" : 0,
        "drop_ios" : 0,
        "slat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "clat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "lat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "bw_min" : 0,
        "bw_max" : 0,
        "bw_agg" : 0.000000,
        "bw_mean" : 0.000000,
        "bw_dev" : 0.000000,
        "bw_samples" : 0,
        "iops_min" : 0,
        "iops_max" : 0,
        "iops_mean" : 0.000000,
        "iops_stddev" : 0.000000,
        "iops_samples" : 0
      },
      "sync" : {
        "total_ios" : 0,
        "lat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        }
      },
      "job_runtime" : 19210274,
      "usr_cpu" : 0.005950,
      "sys_cpu" : 0.030385,
      "ctx" : 446567,
      "majf" : 0,
      "minf" : 883,
      "iodepth_level" : {
        "1" : 0.100000,
        "2" : 0.100000,
        "4" : 0.100000,
        "8" : 0.100000,
        "16" : 0.100000,
        "32" : 99.895970,
        ">=64" : 0.000000
      },
      "iodepth_submit" : {
        "0" : 0.000000,
        "4" : 100.000000,
        "8" : 0.000000,
        "16" : 0.000000,
        "32" : 0.000000,
        "64" : 0.000000,
        ">=64" : 0.000000
      },
      "iodepth_complete" : {
        "0" : 0.000000,
        "4" : 99.996641,
        "8" : 0.000000,
        "16" : 0.000000,
        "32" : 0.100000,
        "64" : 0.000000,
        ">=64" : 0.000000
      },
      "latency_ns" : {
        "2" : 0.000000,
        "4" : 0.000000,
        "10" : 0.000000,
        "20" : 0.000000,
        "50" : 0.000000,
        "100" : 0.000000,
        "250" : 0.000000,
        "500" : 0.000000,
        "750" : 0.000000,
        "1000" : 0.000000
      },
      "latency_us" : {
        "2" : 0.000000,
        "4" : 0.000000,
        "10" : 0.000000,
        "20" : 0.000000,
        "50" : 0.000000,
        "100" : 0.000000,
        "250" : 0.010000,
        "500" : 0.010000,
        "750" : 0.010000,
        "1000" : 0.010000
      },
      "latency_ms" : {
        "2" : 0.010000,
        "4" : 0.033925,
        "10" : 0.247909,
        "20" : 0.190231,
        "50" : 2.933171,
        "100" : 7.702536,
        "250" : 31.258039,
        "500" : 41.803822,
        "750" : 14.596065,
        "1000" : 1.079935,
        "2000" : 0.153056,
        ">=2000" : 0.000000
      },
      "latency_depth" : 32,
      "latency_target" : 0,
      "latency_percentile" : 100.000000,
      "latency_window" : 0
    }
  ],
  "disk_util" : [
    {
      "name" : "sdb",
      "read_ios" : 0,
      "write_ios" : 1013029,
      "read_merges" : 0,
      "write_merges" : 894106,
      "read_ticks" : 0,
      "write_ticks" : 53607123,
      "in_queue" : 53609981,
      "util" : 96.247980
    }
  ]
}

[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Bugtraq]     [Linux OMAP]     [Linux MIPS]     [eCos]     [Asterisk Internet PBX]     [Linux API]

  Powered by Linux