Patch has been updated according to comments from Andrew Jones.
Option -t adds ability to output in standard
machine-readable format - TAP (TestAnythingProtocol).
Example:
$ ./run_tests.sh -t -j5
TAP version 13
ok smptest
ok ioapic-split
ok ioapic
ok smptest3
ok vmexit_cpuid
ok vmexit_vmcall
ok vmexit_mov_from_cr8
ok vmexit_mov_to_cr8
ok vmexit_inl_pmtimer
ok vmexit_ipi
ok vmexit_ipi_halt
...
Signed-off-by: Sergey Bronnikov <sergeyb@xxxxxxxxxxxxxx>
---
run_tests.sh | 21 +++++++++++++++++++--
scripts/runtime.bash | 51 ++++++++++++++++++++++++++++++++++++++++++---------
2 files changed, 61 insertions(+), 11 deletions(-)
diff --git a/run_tests.sh b/run_tests.sh
index aa2e65f..102c806 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
verbose="no"
+tap_output="no"
run_all_tests="no" # don't run nodefault tests
if [ ! -f config.mak ]; then
@@ -14,7 +15,7 @@ function usage()
{
cat <<EOF
-Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS]
+Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS] [-t]
-h: Output this help text
-v: Enables verbose mode
@@ -22,6 +23,7 @@ Usage: $0 [-h] [-v] [-a] [-g group] [-j NUM-TASKS]
and those guarded by errata.
-g: Only execute tests in the given group
-j: Execute tests in parallel
+ -t: Output test results in TAP format
Set the environment variable QEMU=/path/to/qemu-system-ARCH to
specify the appropriate qemu binary for ARCH-run.
@@ -32,7 +34,7 @@ EOF
RUNTIME_arch_run="./$TEST_DIR/run"
source scripts/runtime.bash
-while getopts "ag:hj:v" opt; do
+while getopts "ag:htj:v" opt; do
case $opt in
a)
run_all_tests="yes"
@@ -55,6 +57,9 @@ while getopts "ag:hj:v" opt; do
v)
verbose="yes"
;;
+ t)
+ tap_output="yes"
+ ;;
*)
exit 2
;;
@@ -74,6 +79,10 @@ RUNTIME_log_stdout () {
function run_task()
{
local testname="$1"
+ if [ -z "$testname" ]; then
+ return
+ fi
+ test_number=$((test_number+1))
while (( $(jobs | wc -l) == $unittest_run_queues )); do
# wait for any background test to finish
@@ -98,8 +107,16 @@ mkdir $unittest_log_dir || exit 2
echo "BUILD_HEAD=$(cat build-head)" > $unittest_log_dir/SUMMARY
+if [[ $tap_output == "yes" ]]; then
+ test_number=0
+ echo "TAP version 13"
+fi
trap "wait; exit 130" SIGINT
for_each_unittest $config run_task
# wait until all tasks finish
wait
+
+if [[ $tap_output == "yes" ]]; then
+ echo "1..$test_number"
+fi
diff --git a/scripts/runtime.bash b/scripts/runtime.bash
index a31ae91..45f139e 100644
--- a/scripts/runtime.bash
+++ b/scripts/runtime.bash
@@ -50,6 +50,39 @@ skip_nodefault()
done
}
+function print_result()
+{
+ # output test results in a TAP format
+ # https://testanything.org/tap-version-13-specification.html
+
+ local status="$1"
+ local testname="$2"
+ local summary="$3"
+ local reason="$4"
+
+ if [[ $tap_output == "no" ]]; then
+ if [ -z "$reason" ]; then
+ echo "`$status` $testname $summary"
+ else
+ echo "`$status` $testname ($reason)"
+ fi
+ return
+ fi
+
+ if [[ $status == "FAIL" ]]; then
+ echo -e "not ok $testname $reason"
+ return
+ elif [[ $status == "PASS" ]]; then
+ echo -e "ok $testname"
+ return
+ elif [[ $status == "SKIP" ]]; then
+ echo -e "ok $testname # SKIP $reason"
+ return
+ else
+ echo -e "not ok # TODO unknown test status"
+ fi
+}
+
function run()
{
local testname="$1"
@@ -72,12 +105,12 @@ function run()
if [ -z "$only_group" ] && grep -qw "nodefault" <<<$groups &&
skip_nodefault; then
- echo -e "`SKIP` $testname (test marked as manual run only)"
+ print_result "SKIP" $testname "" "test marked as manual run only"
return;
fi
if [ -n "$arch" ] && [ "$arch" != "$ARCH" ]; then
- echo "`SKIP` $1 ($arch only)"
+ print_result "SKIP" $testname "" "$arch only"
return 2
fi
@@ -88,13 +121,13 @@ function run()
path=${check_param%%=*}
value=${check_param#*=}
if [ "$path" ] && [ "$(cat $path)" != "$value" ]; then
- echo "`SKIP` $1 ($path not equal to $value)"
+ print_result "SKIP" $testname "" "$path not equal to $value"
return 2
fi
done
last_line=$(premature_failure > >(tail -1)) && {
- echo "`SKIP` $1 ($last_line)"
+ print_result "SKIP" $testname "" "$last_line"
return 77
}
@@ -115,15 +148,15 @@ function run()
[ "$STANDALONE" != "yes" ] && echo > >(RUNTIME_log_stdout $kernel)
if [ $ret -eq 0 ]; then
- echo "`PASS` $1 $summary"
+ print_result "PASS" $testname "$summary"
elif [ $ret -eq 77 ]; then
- echo "`SKIP` $1 $summary"
+ print_result "SKIP" $testname "$summary"
elif [ $ret -eq 124 ]; then
- echo "`FAIL` $1 (timeout; duration=$timeout)"
+ print_result "FAIL" $testname "" "timeout; duration=$timeout"
elif [ $ret -gt 127 ]; then
- echo "`FAIL` $1 (terminated on SIG$(kill -l $(($ret - 128))))"
+ print_result "FAIL" $testname "" "terminated on SIG$(kill -l $(($ret - 128)))"
else
- echo "`FAIL` $1 $summary"
+ print_result "FAIL" $testname "$summary"
fi
return $ret
--
2.16.2
