On Tue, Jul 17, 2018 at 08:35:02AM +0300, Sergey Bronnikov wrote:
> Patch has been updated according to comments from Andrew Jones.
The above line should be below the '---', as it doesn't need to
get into the commit history.
>
> Option -t adds ability to output in standard
> machine-readable format - TAP (TestAnythingProtocol).
>
> Example:
>
> $ ./run_tests.sh -t -j5
> TAP version 13
> ok smptest
> ok ioapic-split
> ok ioapic
> ok smptest3
> ok vmexit_cpuid
> ok vmexit_vmcall
> ok vmexit_mov_from_cr8
> ok vmexit_mov_to_cr8
> ok vmexit_inl_pmtimer
> ok vmexit_ipi
> ok vmexit_ipi_halt
> ...
>
> Signed-off-by: Sergey Bronnikov <sergeyb@xxxxxxxxxxxxxx>
> ---
> run_tests.sh | 21 +++++++++++++++++++--
> scripts/runtime.bash | 51 ++++++++++++++++++++++++++++++++++++++++++---------
> 2 files changed, 61 insertions(+), 11 deletions(-)
>
> diff --git a/run_tests.sh b/run_tests.sh
> index aa2e65f..102c806 100755
> --- a/run_tests.sh
> +++ b/run_tests.sh
> @@ -1,6 +1,7 @@
> #!/usr/bin/env bash
>
> verbose="no"
> +tap_output="no"
> run_all_tests="no" # don't run nodefault tests
>
> if [ ! -f config.mak ]; then
> @@ -14,7 +15,7 @@ function usage()
> {
> cat <<EOF
>
> -Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS]
> +Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS] [-t]
>
> -h: Output this help text
> -v: Enables verbose mode
> @@ -22,6 +23,7 @@ Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS]
> and those guarded by errata.
> -g: Only execute tests in the given group
> -j: Execute tests in parallel
> + -t: Output test results in TAP format
>
> Set the environment variable QEMU=/path/to/qemu-system-ARCH to
> specify the appropriate qemu binary for ARCH-run.
> @@ -32,7 +34,7 @@ EOF
> RUNTIME_arch_run="./$TEST_DIR/run"
> source scripts/runtime.bash
>
> -while getopts "ag:hj:v" opt; do
> +while getopts "ag:htj:v" opt; do
> case $opt in
> a)
> run_all_tests="yes"
> @@ -55,6 +57,9 @@ while getopts "ag:hj:v" opt; do
> v)
> verbose="yes"
> ;;
> + t)
> + tap_output="yes"
> + ;;
> *)
> exit 2
> ;;
> @@ -74,6 +79,10 @@ RUNTIME_log_stdout () {
> function run_task()
> {
> local testname="$1"
> + if [ -z "$testname" ]; then
> + return
> + fi
> + test_number=$((test_number+1))
>
> while (( $(jobs | wc -l) == $unittest_run_queues )); do
> # wait for any background test to finish
> @@ -98,8 +107,16 @@ mkdir $unittest_log_dir || exit 2
>
> echo "BUILD_HEAD=$(cat build-head)" > $unittest_log_dir/SUMMARY
>
> +if [[ $tap_output == "yes" ]]; then
> + test_number=0
> + echo "TAP version 13"
> +fi
> trap "wait; exit 130" SIGINT
> for_each_unittest $config run_task
>
> # wait until all tasks finish
> wait
> +
> +if [[ $tap_output == "yes" ]]; then
> + echo "1..$test_number"
> +fi
> diff --git a/scripts/runtime.bash b/scripts/runtime.bash
> index a31ae91..45f139e 100644
> --- a/scripts/runtime.bash
> +++ b/scripts/runtime.bash
> @@ -50,6 +50,39 @@ skip_nodefault()
> done
> }
>
> +function print_result()
> +{
> + # output test results in a TAP format
> + # https://testanything.org/tap-version-13-specification.html
> +
> + local status="$1"
> + local testname="$2"
> + local summary="$3"
> + local reason="$4"
> +
> + if [[ $tap_output == "no" ]]; then
> + if [ -z "$reason" ]; then
> + echo "`$status` $testname $summary"
> + else
> + echo "`$status` $testname ($reason)"
> + fi
> + return
> + fi
> +
When I pointed out the unneeded return's I meant all of
them in the if-elif chain below. None of the returns
(except the one above) are needed in this function.
> + if [[ $status == "FAIL" ]]; then
> + echo -e "not ok $testname $reason"
> + return
> + elif [[ $status == "PASS" ]]; then
> + echo -e "ok $testname"
> + return
> + elif [[ $status == "SKIP" ]]; then
> + echo -e "ok $testname # SKIP $reason"
> + return
> + else
> + echo -e "not ok # TODO unknown test status"
Why are we using '-e' in the four echo statements above? I
don't think we want to be.
> + fi
> +}
> +
> function run()
> {
> local testname="$1"
> @@ -72,12 +105,12 @@ function run()
>
> if [ -z "$only_group" ] && grep -qw "nodefault" <<<$groups &&
> skip_nodefault; then
> - echo -e "`SKIP` $testname (test marked as manual run only)"
> + print_result "SKIP" $testname "" "test marked as manual run only"
> return;
> fi
>
> if [ -n "$arch" ] && [ "$arch" != "$ARCH" ]; then
> - echo "`SKIP` $1 ($arch only)"
> + print_result "SKIP" $testname "" "$arch only"
> return 2
> fi
>
> @@ -88,13 +121,13 @@ function run()
> path=${check_param%%=*}
> value=${check_param#*=}
> if [ "$path" ] && [ "$(cat $path)" != "$value" ]; then
> - echo "`SKIP` $1 ($path not equal to $value)"
> + print_result "SKIP" $testname "" "$path not equal to $value"
> return 2
> fi
> done
>
> last_line=$(premature_failure > >(tail -1)) && {
> - echo "`SKIP` $1 ($last_line)"
> + print_result "SKIP" $testname "" "$last_line"
> return 77
> }
>
> @@ -115,15 +148,15 @@ function run()
> [ "$STANDALONE" != "yes" ] && echo > >(RUNTIME_log_stdout $kernel)
>
> if [ $ret -eq 0 ]; then
> - echo "`PASS` $1 $summary"
> + print_result "PASS" $testname "$summary"
> elif [ $ret -eq 77 ]; then
> - echo "`SKIP` $1 $summary"
> + print_result "SKIP" $testname "$summary"
> elif [ $ret -eq 124 ]; then
> - echo "`FAIL` $1 (timeout; duration=$timeout)"
> + print_result "FAIL" $testname "" "timeout; duration=$timeout"
> elif [ $ret -gt 127 ]; then
> - echo "`FAIL` $1 (terminated on SIG$(kill -l $(($ret - 128))))"
> + print_result "FAIL" $testname "" "terminated on SIG$(kill -l $(($ret - 128)))"
> else
> - echo "`FAIL` $1 $summary"
> + print_result "FAIL" $testname "$summary"
> fi
>
> return $ret
> --
> 2.16.2
Thanks,
drew
