The following changes since commit 26949a8bbc544094740c61f79b4ea778978e7d8e: Merge branch '32-bit-fixes' of https://github.com/sitsofe/fio (2020-05-19 16:14:19 -0600) are available in the Git repository at: git://git.kernel.dk/fio.git master for you to fetch changes up to 0e21f5c6f64a73bcede20e1a29a885845e453b8e: Merge branch 'testing' of https://github.com/vincentkfu/fio (2020-05-20 14:19:12 -0600) ---------------------------------------------------------------- Jens Axboe (2): Merge branch 'issue-989' of https://github.com/Nordix/fio Merge branch 'testing' of https://github.com/vincentkfu/fio Lars Ekman (1): Corrected scope of for-loop Vincent Fu (6): t/jsonplus2csv_test: reduce file size t/run-fio-tests: better catch file errors t/latency_percentiles: run cmdprio_percentage tests only if root docs: update cmdprio_percentage with note about root user testing: use max-jobs to speed up testing t/zbd: improve error handling for test scripts HOWTO | 3 ++- fio.1 | 3 ++- t/jsonplus2csv_test.py | 5 ++++- t/latency_percentiles.py | 5 +++-- t/readonly.py | 1 + t/run-fio-tests.py | 37 +++++++++++++++++++++++------------ t/steadystate_tests.py | 2 +- t/strided.py | 1 + t/zbd/run-tests-against-regular-nullb | 4 ++-- t/zbd/run-tests-against-zoned-nullb | 2 +- tools/plot/fio2gnuplot | 8 ++++---- 11 files changed, 45 insertions(+), 26 deletions(-) --- Diff of recent changes: diff --git a/HOWTO b/HOWTO index 430c7b62..cd628552 100644 --- a/HOWTO +++ b/HOWTO @@ -2040,7 +2040,8 @@ with the caveat that when used on the command line, they must come after the the priority bit. Non-read I/O is likely unaffected by ``cmdprio_percentage``. This option cannot be used with the `prio` or `prioclass` options. For this option to set the priority bit properly, NCQ priority must be supported and - enabled and :option:`direct`\=1 option must be used. + enabled and :option:`direct`\=1 option must be used. fio must also be run as + the root user. .. option:: fixedbufs : [io_uring] diff --git a/fio.1 b/fio.1 index a2379f98..9e9e1b1a 100644 --- a/fio.1 +++ b/fio.1 @@ -1806,7 +1806,8 @@ Set the percentage of I/O that will be issued with higher priority by setting the priority bit. Non-read I/O is likely unaffected by ``cmdprio_percentage``. This option cannot be used with the `prio` or `prioclass` options. For this option to set the priority bit properly, NCQ priority must be supported and -enabled and `direct=1' option must be used. +enabled and `direct=1' option must be used. fio must also be run as the root +user. .TP .BI (io_uring)fixedbufs If fio is asked to do direct IO, then Linux will map pages for each IO call, and diff --git a/t/jsonplus2csv_test.py b/t/jsonplus2csv_test.py index 2b34ef25..f01a9f32 100755 --- a/t/jsonplus2csv_test.py +++ b/t/jsonplus2csv_test.py @@ -44,6 +44,7 @@ def run_fio(fio): fio path to fio executable. """ +# We need an async ioengine to get submission latencies if platform.system() == 'Linux': aio = 'libaio' elif platform.system() == 'Windows': @@ -52,13 +53,14 @@ def run_fio(fio): aio = 'posixaio' fio_args = [ + "--max-jobs=4", "--output=fio-output.json", "--output-format=json+", "--filename=fio_jsonplus_clat2csv.test", "--ioengine=" + aio, "--time_based", "--runtime=3s", - "--size=1G", + "--size=1M", "--slat_percentiles=1", "--clat_percentiles=1", "--lat_percentiles=1", @@ -87,6 +89,7 @@ def check_output(fio_output, script_path): """ if fio_output.returncode != 0: + print("ERROR: fio run failed") return False if platform.system() == 'Windows': diff --git a/t/latency_percentiles.py b/t/latency_percentiles.py index 5cdd49cf..6ce4579a 100755 --- a/t/latency_percentiles.py +++ b/t/latency_percentiles.py @@ -109,6 +109,7 @@ class FioLatTest(): """Run a test.""" fio_args = [ + "--max-jobs=16", "--name=latency", "--randrepeat=0", "--norandommap", @@ -1303,9 +1304,9 @@ def main(): (args.run_only and test['test_id'] not in args.run_only): skipped = skipped + 1 outcome = 'SKIPPED (User request)' - elif platform.system() != 'Linux' and 'cmdprio_percentage' in test: + elif (platform.system() != 'Linux' or os.geteuid() != 0) and 'cmdprio_percentage' in test: skipped = skipped + 1 - outcome = 'SKIPPED (Linux required for cmdprio_percentage tests)' + outcome = 'SKIPPED (Linux root required for cmdprio_percentage tests)' else: test_obj = test['test_obj'](artifact_root, test, args.debug) status = test_obj.run_fio(fio) diff --git a/t/readonly.py b/t/readonly.py index 43686c9c..464847c6 100755 --- a/t/readonly.py +++ b/t/readonly.py @@ -36,6 +36,7 @@ def parse_args(): def run_fio(fio, test, index): fio_args = [ + "--max-jobs=16", "--name=readonly", "--ioengine=null", "--time_based", diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py index 8e326ed5..763e0103 100755 --- a/t/run-fio-tests.py +++ b/t/run-fio-tests.py @@ -226,6 +226,7 @@ class FioJobTest(FioExeTest): self.json_data = None self.fio_output = "{0}.output".format(os.path.basename(self.fio_job)) self.fio_args = [ + "--max-jobs=16", "--output-format={0}".format(self.output_format), "--output={0}".format(self.fio_output), self.fio_job, @@ -273,6 +274,20 @@ class FioJobTest(FioExeTest): else: logging.debug("Test %d: precondition step failed", self.testnum) + @classmethod + def get_file(cls, filename): + """Safely read a file.""" + file_data = '' + success = True + + try: + with open(filename, "r") as output_file: + file_data = output_file.read() + except OSError: + success = False + + return file_data, success + def check_result(self): """Check fio job results.""" @@ -289,10 +304,8 @@ class FioJobTest(FioExeTest): if 'json' not in self.output_format: return - try: - with open(os.path.join(self.test_dir, self.fio_output), "r") as output_file: - file_data = output_file.read() - except EnvironmentError: + file_data, success = self.get_file(os.path.join(self.test_dir, self.fio_output)) + if not success: self.failure_reason = "{0} unable to open output file,".format(self.failure_reason) self.passed = False return @@ -457,11 +470,9 @@ class Requirements(object): Requirements._linux = platform.system() == "Linux" if Requirements._linux: - try: - config_file = os.path.join(fio_root, "config-host.h") - with open(config_file, "r") as config: - contents = config.read() - except Exception: + config_file = os.path.join(fio_root, "config-host.h") + contents, success = FioJobTest.get_file(config_file) + if not success: print("Unable to open {0} to check requirements".format(config_file)) Requirements._zbd = True else: @@ -899,10 +910,10 @@ def main(): else: result = "FAILED: {0}".format(test.failure_reason) failed = failed + 1 - with open(test.stderr_file, "r") as stderr_file: - logging.debug("Test %d: stderr:\n%s", config['test_id'], stderr_file.read()) - with open(test.stdout_file, "r") as stdout_file: - logging.debug("Test %d: stdout:\n%s", config['test_id'], stdout_file.read()) + contents, _ = FioJobTest.get_file(test.stderr_file) + logging.debug("Test %d: stderr:\n%s", config['test_id'], contents) + contents, _ = FioJobTest.get_file(test.stdout_file) + logging.debug("Test %d: stdout:\n%s", config['test_id'], contents) print("Test {0} {1}".format(config['test_id'], result)) print("{0} test(s) passed, {1} failed, {2} skipped".format(passed, failed, skipped)) diff --git a/t/steadystate_tests.py b/t/steadystate_tests.py index b55a67ac..e99b655c 100755 --- a/t/steadystate_tests.py +++ b/t/steadystate_tests.py @@ -122,7 +122,7 @@ if __name__ == '__main__': for job in reads: tf = "steadystate_job{0}.json".format(jobnum) - parameters = [ "--name=job{0}".format(jobnum) ] + parameters = [ "--max-jobs=16", "--name=job{0}".format(jobnum) ] parameters.extend([ "--thread", "--output-format=json", "--output={0}".format(tf), diff --git a/t/strided.py b/t/strided.py index aac15d10..6d34be8a 100755 --- a/t/strided.py +++ b/t/strided.py @@ -52,6 +52,7 @@ def parse_args(): def run_fio(fio, test, index): filename = "strided" fio_args = [ + "--max-jobs=16", "--name=strided", "--zonemode=strided", "--log_offset=1", diff --git a/t/zbd/run-tests-against-regular-nullb b/t/zbd/run-tests-against-regular-nullb index 0f6e4b66..5b7b4009 100755 --- a/t/zbd/run-tests-against-regular-nullb +++ b/t/zbd/run-tests-against-regular-nullb @@ -8,7 +8,7 @@ scriptdir="$(cd "$(dirname "$0")" && pwd)" for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d"; done modprobe -r null_blk -modprobe null_blk nr_devices=0 || return $? +modprobe null_blk nr_devices=0 || exit $? for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d" done @@ -22,6 +22,6 @@ modprobe null_blk nr_devices=0 && echo 4096 > blocksize && echo 1024 > size && echo 1 > memory_backed && - echo 1 > power + echo 1 > power || exit $? "${scriptdir}"/test-zbd-support "$@" /dev/nullb0 diff --git a/t/zbd/run-tests-against-zoned-nullb b/t/zbd/run-tests-against-zoned-nullb index 0952011c..53aee3e8 100755 --- a/t/zbd/run-tests-against-zoned-nullb +++ b/t/zbd/run-tests-against-zoned-nullb @@ -8,7 +8,7 @@ scriptdir="$(cd "$(dirname "$0")" && pwd)" for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d"; done modprobe -r null_blk -modprobe null_blk nr_devices=0 || return $? +modprobe null_blk nr_devices=0 || exit $? for d in /sys/kernel/config/nullb/*; do [ -d "$d" ] && rmdir "$d" done diff --git a/tools/plot/fio2gnuplot b/tools/plot/fio2gnuplot index cc4ea4c7..69aa791e 100755 --- a/tools/plot/fio2gnuplot +++ b/tools/plot/fio2gnuplot @@ -93,10 +93,10 @@ set style line 1 lt 1 lw 3 pt 3 linecolor rgb "green" compare_smooth.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); compare_trend.write("plot %s w l ls 1 ti 'Global average value (%.2f)'" % (global_avg,global_avg)); - pos=0 - # Let's create a temporary file for each selected fio file - for file in fio_data_file: - tmp_filename = "gnuplot_temp_file.%d" % pos + pos=0 + # Let's create a temporary file for each selected fio file + for file in fio_data_file: + tmp_filename = "gnuplot_temp_file.%d" % pos # Plotting comparing graphs doesn't have a meaning unless if there is at least 2 traces if len(fio_data_file) > 1:
