The following changes since commit c96b385b6e0c78478697713e6da9174fba2432d3: t/zbd: make the test script easier to terminate (2020-05-25 18:21:45 -0600) are available in the Git repository at: git://git.kernel.dk/fio.git master for you to fetch changes up to dacac3b936092aa73e29257fefcb7bf5658ec7cb: Merge branch 'python3-testing' of https://github.com/vincentkfu/fio (2020-05-28 13:27:15 -0600) ---------------------------------------------------------------- Jens Axboe (2): Merge branch 'testing' of https://github.com/vincentkfu/fio Merge branch 'python3-testing' of https://github.com/vincentkfu/fio Shin'ichiro Kawasaki (2): pshared: Add mutex_init_pshared_with_type() zbd: Fix compilation error on BSD Vincent Fu (5): appveyor: use on_finish section to upload artifacts t/run-fio-tests: pass-through arguments to test scripts .travis: enable arm64 architecture builds testing: change two test scripts to refer to python3 travis: install python3 scipy for Linux and macOS tests .appveyor.yml | 6 +++--- .travis.yml | 26 ++++++++++++++++++-------- pshared.c | 15 ++++++++++++++- pshared.h | 1 + t/run-fio-tests.py | 27 ++++++++++++++++++--------- t/steadystate_tests.py | 3 +-- t/strided.py | 3 +-- zbd.c | 22 +++++++--------------- 8 files changed, 63 insertions(+), 40 deletions(-) --- Diff of recent changes: diff --git a/.appveyor.yml b/.appveyor.yml index e2351be7..70c337f8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -26,10 +26,10 @@ after_build: test_script: - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && file.exe fio.exe && make.exe test' - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -f fio.exe ] && python.exe t/run-fio-tests.py --artifact-root test-artifacts --debug' - - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -d test-artifacts ] && 7z a -t7z test-artifacts.7z test-artifacts -xr!foo.0.0 -xr!latency.?.0 -xr!fio_jsonplus_clat2csv.test' artifacts: - path: os\windows\*.msi name: msi - - path: test-artifacts.7z - name: test-artifacts + +on_finish: + - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && [ -d test-artifacts ] && 7z a -t7z test-artifacts.7z test-artifacts -xr!foo.0.0 -xr!latency.?.0 -xr!fio_jsonplus_clat2csv.test && appveyor PushArtifact test-artifacts.7z' diff --git a/.travis.yml b/.travis.yml index 77c31b77..eba16baa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ os: compiler: - clang - gcc +arch: + - amd64 + - arm64 env: matrix: - BUILD_ARCH="x86" @@ -17,28 +20,34 @@ matrix: - os: osx compiler: clang # Workaround travis setting CC=["clang", "gcc"] env: BUILD_ARCH="x86_64" + arch: amd64 # Latest xcode image (needs periodic updating) - os: osx compiler: clang osx_image: xcode11.2 env: BUILD_ARCH="x86_64" + arch: amd64 exclude: - os: osx compiler: gcc - exclude: - os: linux compiler: clang + arch: amd64 env: BUILD_ARCH="x86" # Only do the gcc x86 build to reduce clutter + - os: linux + env: BUILD_ARCH="x86" + arch: arm64 + before_install: - EXTRA_CFLAGS="-Werror" - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pkgs=(libaio-dev libcunit1 libcunit1-dev libgoogle-perftools4 libibverbs-dev libiscsi-dev libnuma-dev librbd-dev librdmacm-dev libz-dev); if [[ "$BUILD_ARCH" == "x86" ]]; then pkgs=("${pkgs[@]/%/:i386}"); - pkgs+=(gcc-multilib python-scipy); + pkgs+=(gcc-multilib python3-scipy); EXTRA_CFLAGS="${EXTRA_CFLAGS} -m32"; else - pkgs+=(glusterfs-common python-scipy); + pkgs+=(glusterfs-common python3-scipy); fi; sudo apt-get -qq update; sudo apt-get install --no-install-recommends -qq -y "${pkgs[@]}"; @@ -46,12 +55,13 @@ before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; brew install cunit; - if [[ "$TRAVIS_OSX_IMAGE" == "xcode11.2" ]]; then - pip3 install scipy; - fi; - pip install scipy; + pip3 install scipy; fi; script: - ./configure --extra-cflags="${EXTRA_CFLAGS}" && make - make test - - sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug + - if [[ "$TRAVIS_CPU_ARCH" == "arm64" ]]; then + sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug -p 1010:"--skip 15 16 17 18 19 20"; + else + sudo python3 t/run-fio-tests.py --skip 6 1007 1008 --debug; + fi; diff --git a/pshared.c b/pshared.c index 21192556..791faf95 100644 --- a/pshared.c +++ b/pshared.c @@ -39,7 +39,7 @@ int cond_init_pshared(pthread_cond_t *cond) return 0; } -int mutex_init_pshared(pthread_mutex_t *mutex) +int mutex_init_pshared_with_type(pthread_mutex_t *mutex, int type) { pthread_mutexattr_t mattr; int ret; @@ -60,6 +60,14 @@ int mutex_init_pshared(pthread_mutex_t *mutex) return ret; } #endif + if (type) { + ret = pthread_mutexattr_settype(&mattr, type); + if (ret) { + log_err("pthread_mutexattr_settype: %s\n", + strerror(ret)); + return ret; + } + } ret = pthread_mutex_init(mutex, &mattr); if (ret) { log_err("pthread_mutex_init: %s\n", strerror(ret)); @@ -69,6 +77,11 @@ int mutex_init_pshared(pthread_mutex_t *mutex) return 0; } +int mutex_init_pshared(pthread_mutex_t *mutex) +{ + return mutex_init_pshared_with_type(mutex, 0); +} + int mutex_cond_init_pshared(pthread_mutex_t *mutex, pthread_cond_t *cond) { int ret; diff --git a/pshared.h b/pshared.h index a58df6fe..f33be462 100644 --- a/pshared.h +++ b/pshared.h @@ -3,6 +3,7 @@ #include <pthread.h> +extern int mutex_init_pshared_with_type(pthread_mutex_t *, int); extern int mutex_init_pshared(pthread_mutex_t *); extern int cond_init_pshared(pthread_cond_t *); extern int mutex_cond_init_pshared(pthread_mutex_t *, pthread_cond_t *); diff --git a/t/run-fio-tests.py b/t/run-fio-tests.py index 763e0103..e7063d3e 100755 --- a/t/run-fio-tests.py +++ b/t/run-fio-tests.py @@ -122,10 +122,7 @@ class FioExeTest(FioTest): def run(self): """Execute the binary or script described by this instance.""" - if self.parameters: - command = [self.exe_path] + self.parameters - else: - command = [self.exe_path] + command = [self.exe_path] + self.parameters command_file = open(self.command_file, "w+") command_file.write("%s\n" % command) command_file.close() @@ -797,6 +794,8 @@ def parse_args(): help='provide debug output') parser.add_argument('-k', '--skip-req', action='store_true', help='skip requirements checking') + parser.add_argument('-p', '--pass-through', action='append', + help='pass-through an argument to an executable test') args = parser.parse_args() return args @@ -811,6 +810,17 @@ def main(): else: logging.basicConfig(level=logging.INFO) + pass_through = {} + if args.pass_through: + for arg in args.pass_through: + if not ':' in arg: + print("Invalid --pass-through argument '%s'" % arg) + print("Syntax for --pass-through is TESTNUMBER:ARGUMENT") + return + split = arg.split(":",1) + pass_through[int(split[0])] = split[1] + logging.debug("Pass-through arguments: %s" % pass_through) + if args.fio_root: fio_root = args.fio_root else: @@ -874,13 +884,12 @@ def main(): if config['parameters']: parameters = [p.format(fio_path=fio_path) for p in config['parameters']] else: - parameters = None + parameters = [] if Path(exe_path).suffix == '.py' and platform.system() == "Windows": - if parameters: - parameters.insert(0, exe_path) - else: - parameters = [exe_path] + parameters.insert(0, exe_path) exe_path = "python.exe" + if config['test_id'] in pass_through: + parameters += pass_through[config['test_id']].split() test = config['test_class'](exe_path, parameters, config['success']) else: diff --git a/t/steadystate_tests.py b/t/steadystate_tests.py index e99b655c..e8bd768c 100755 --- a/t/steadystate_tests.py +++ b/t/steadystate_tests.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# Note: this script is python2 and python3 compatible. +#!/usr/bin/env python3 # # steadystate_tests.py # diff --git a/t/strided.py b/t/strided.py index 6d34be8a..45e6f148 100755 --- a/t/strided.py +++ b/t/strided.py @@ -1,5 +1,4 @@ -#!/usr/bin/python -# Note: this script is python2 and python3 compatible. +#!/usr/bin/env python3 # # strided.py # diff --git a/zbd.c b/zbd.c index 72352db0..a7572c9a 100644 --- a/zbd.c +++ b/zbd.c @@ -19,6 +19,7 @@ #include "oslib/asprintf.h" #include "smalloc.h" #include "verify.h" +#include "pshared.h" #include "zbd.h" /** @@ -353,7 +354,6 @@ static int init_zone_info(struct thread_data *td, struct fio_file *f) struct fio_zone_info *p; uint64_t zone_size = td->o.zone_size; struct zoned_block_device_info *zbd_info = NULL; - pthread_mutexattr_t attr; int i; if (zone_size == 0) { @@ -374,14 +374,12 @@ static int init_zone_info(struct thread_data *td, struct fio_file *f) if (!zbd_info) return -ENOMEM; - pthread_mutexattr_init(&attr); - pthread_mutexattr_setpshared(&attr, true); - pthread_mutex_init(&zbd_info->mutex, &attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + mutex_init_pshared(&zbd_info->mutex); zbd_info->refcount = 1; p = &zbd_info->zone_info[0]; for (i = 0; i < nr_zones; i++, p++) { - pthread_mutex_init(&p->mutex, &attr); + mutex_init_pshared_with_type(&p->mutex, + PTHREAD_MUTEX_RECURSIVE); p->start = i * zone_size; p->wp = p->start + zone_size; p->type = ZBD_ZONE_TYPE_SWR; @@ -395,7 +393,6 @@ static int init_zone_info(struct thread_data *td, struct fio_file *f) f->zbd_info->zone_size_log2 = is_power_of_2(zone_size) ? ilog2(zone_size) : 0; f->zbd_info->nr_zones = nr_zones; - pthread_mutexattr_destroy(&attr); return 0; } @@ -415,12 +412,8 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) struct fio_zone_info *p; uint64_t zone_size, offset; struct zoned_block_device_info *zbd_info = NULL; - pthread_mutexattr_t attr; int i, j, ret = 0; - pthread_mutexattr_init(&attr); - pthread_mutexattr_setpshared(&attr, true); - zones = calloc(ZBD_REPORT_MAX_ZONES, sizeof(struct zbd_zone)); if (!zones) goto out; @@ -454,14 +447,14 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) ret = -ENOMEM; if (!zbd_info) goto out; - pthread_mutex_init(&zbd_info->mutex, &attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + mutex_init_pshared(&zbd_info->mutex); zbd_info->refcount = 1; p = &zbd_info->zone_info[0]; for (offset = 0, j = 0; j < nr_zones;) { z = &zones[0]; for (i = 0; i < nrz; i++, j++, z++, p++) { - pthread_mutex_init(&p->mutex, &attr); + mutex_init_pshared_with_type(&p->mutex, + PTHREAD_MUTEX_RECURSIVE); p->start = z->start; switch (z->cond) { case ZBD_ZONE_COND_NOT_WP: @@ -512,7 +505,6 @@ static int parse_zone_info(struct thread_data *td, struct fio_file *f) out: sfree(zbd_info); free(zones); - pthread_mutexattr_destroy(&attr); return ret; }
