Hi Sergey, I like this idea. We may even want to consider changing report*() functions. Some comments below. On Sat, Jul 14, 2018 at 12:59:12AM +0300, Sergey Bronnikov wrote: > Hello! > > By default kvm-unit tests reports results in a simple but non-standard > format. Patch below adds TAP support to kvm-unit tests. TAP is de-facto > standard among other test results formats. See https://testanything.org/. > > $ ./run_tests.sh -t -j5 > TAP version 13 > ok smptest > ok ioapic-split > ok ioapic > ok smptest3 > ok vmexit_cpuid > ok vmexit_vmcall > ok vmexit_mov_from_cr8 > ok vmexit_mov_to_cr8 > ok vmexit_inl_pmtimer > ok vmexit_ipi > ok vmexit_ipi_halt > ... > > Signed-off-by: Sergey Bronnikov <sergeyb@xxxxxxxxxxxxxx> > --- > run_tests.sh | 21 +++++++++++++++++++-- > scripts/runtime.bash | 52 +++++++++++++++++++++++++++++++++++++++++++--------- > 2 files changed, 62 insertions(+), 11 deletions(-) > > diff --git a/run_tests.sh b/run_tests.sh > index aa2e65f..af6d9d3 100755 > --- a/run_tests.sh > +++ b/run_tests.sh > @@ -1,6 +1,7 @@ > #!/usr/bin/env bash > > verbose="no" > +tap_output="no" > run_all_tests="no" # don't run nodefault tests > > if [ ! -f config.mak ]; then > @@ -14,7 +15,7 @@ function usage() > { > cat <<EOF > > -Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS] > +Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS] [-t] > > -h: Output this help text > -v: Enables verbose mode > @@ -22,6 +23,7 @@ Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS] > and those guarded by errata. > -g: Only execute tests in the given group > -j: Execute tests in parallel > + -t: Output test results in TAP format > > Set the environment variable QEMU=/path/to/qemu-system-ARCH to > specify the appropriate qemu binary for ARCH-run. > @@ -32,7 +34,7 @@ EOF > RUNTIME_arch_run="./$TEST_DIR/run" > source scripts/runtime.bash > > -while getopts "ag:hj:v" opt; do > +while getopts "ag:htj:v" opt; do > case $opt in > a) > run_all_tests="yes" > @@ -55,6 +57,9 @@ while getopts "ag:hj:v" opt; do > v) > verbose="yes" > ;; > + t) > + tap_output="yes" > + ;; > *) > exit 2 > ;; > @@ -74,6 +79,10 @@ RUNTIME_log_stdout () { > function run_task() > { > local testname="$1" > + if [ -z "$testname" ]; then > + return > + fi This is an unrelated fix, but OK. > + test_number=$((test_number+1)) This file uses tabs, not spaces for indentation. Your changes should use consistent style. > > while (( $(jobs | wc -l) == $unittest_run_queues )); do > # wait for any background test to finish > @@ -98,8 +107,16 @@ mkdir $unittest_log_dir || exit 2 > > echo "BUILD_HEAD=$(cat build-head)" > $unittest_log_dir/SUMMARY > > +if [ $tap_output == "yes" ]; then To use ==, you should use [[...]]. Or write it as [ "$tap_output" = "yes" ] > + test_number=0 > + echo "TAP version 13" > +fi > trap "wait; exit 130" SIGINT > for_each_unittest $config run_task > > # wait until all tasks finish > wait > + > +if [ $tap_output == "yes" ]; then Same comment as above. > + echo "1..$test_number" > +fi > diff --git a/scripts/runtime.bash b/scripts/runtime.bash > index a31ae91..be4cfbd 100644 > --- a/scripts/runtime.bash > +++ b/scripts/runtime.bash > @@ -50,6 +50,40 @@ skip_nodefault() > done > } > > +function print_result() > +{ > + # output test results in a TAP format > + # https://testanything.org/tap-version-13-specification.html > + > + local status="$1" > + local testname="$2" > + local summary="$3" > + local reason="$4" > + > + if [ $tap_output == "no" ]; then Same comment as above and for the three status checks below. > + if [ -z "$reason" ]; then > + echo "`$status` $testname $summary" > + else > + echo "`$status` $testname ($reason)" > + fi > + return > + fi > + > + if [ $status == "FAIL" ]; then > + echo -e "not ok $testname $reason" > + return > + elif [ $status == "PASS" ]; then > + echo -e "ok $testname" > + return > + elif [ $status == "SKIP" ]; then > + echo -e "ok $testname # SKIP $reason" > + return > + else > + echo -e "not ok # TODO unknown test status" > + return > + fi No need for these 'return's above. > +} > + > function run() > { > local testname="$1" > @@ -72,12 +106,12 @@ function run() > > if [ -z "$only_group" ] && grep -qw "nodefault" <<<$groups && > skip_nodefault; then > - echo -e "`SKIP` $testname (test marked as manual run only)" > + print_result "SKIP" $testname "" "test marked as manual run only" > return; > fi > > if [ -n "$arch" ] && [ "$arch" != "$ARCH" ]; then > - echo "`SKIP` $1 ($arch only)" > + print_result "SKIP" $testname "" "$arch only" > return 2 > fi > > @@ -88,13 +122,13 @@ function run() > path=${check_param%%=*} > value=${check_param#*=} > if [ "$path" ] && [ "$(cat $path)" != "$value" ]; then > - echo "`SKIP` $1 ($path not equal to $value)" > + print_result "SKIP" $testname "" "$path not equal to $value" > return 2 > fi > done > > last_line=$(premature_failure > >(tail -1)) && { > - echo "`SKIP` $1 ($last_line)" > + print_result "SKIP" $testname "" "$last_line" > return 77 > } > > @@ -115,15 +149,15 @@ function run() > [ "$STANDALONE" != "yes" ] && echo > >(RUNTIME_log_stdout $kernel) > > if [ $ret -eq 0 ]; then > - echo "`PASS` $1 $summary" > + print_result "PASS" $testname "$summary" > elif [ $ret -eq 77 ]; then > - echo "`SKIP` $1 $summary" > + print_result "SKIP" $testname "$summary" > elif [ $ret -eq 124 ]; then > - echo "`FAIL` $1 (timeout; duration=$timeout)" > + print_result "FAIL" $testname "" "timeout; duration=$timeout" > elif [ $ret -gt 127 ]; then > - echo "`FAIL` $1 (terminated on SIG$(kill -l $(($ret - 128))))" > + print_result "FAIL" $testname "" "terminated on SIG$(kill -l $(($ret - 128)))" > else > - echo "`FAIL` $1 $summary" > + print_result "FAIL" $testname "$summary" > fi > > return $ret > -- > 2.16.2 Thanks, drew
