The user can set NFT_TEST_SHUFFLE_TESTS=y|n to have the tests shuffled
randomly. The purpose of shuffling is to find tests that depend on each
other, or would break when run in unexpected order.
If unspecified, by default tests are shuffled if no tests are selected
on the command line.
Signed-off-by: Thomas Haller <thaller@xxxxxxxxxx>
---
tests/shell/run-tests.sh | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/tests/shell/run-tests.sh b/tests/shell/run-tests.sh
index 2acea85e7836..a66f32abfc5a 100755
--- a/tests/shell/run-tests.sh
+++ b/tests/shell/run-tests.sh
@@ -143,6 +143,7 @@ usage() {
echo " -U|--no-unshare : Sets NFT_TEST_UNSHARE_CMD=\"\"."
echo " -k|--keep-logs : Sets NFT_TEST_KEEP_LOGS=y."
echo " -s|--sequential : Sets NFT_TEST_JOBS=0, which also enables global cleanups."
+ echo " Also sets NFT_TEST_SHUFFLE_TESTS=n if left unspecified."
echo " -Q|--quick : Sets NFT_TEST_SKIP_slow=y."
echo " -- : Separate options from tests."
echo " [TESTS...] : Other options are treated as test names,"
@@ -198,6 +199,9 @@ usage() {
echo " NFT_TEST_RANDOM_SEED=<SEED>: The test runner will export the environment variable NFT_TEST_RANDOM_SEED"
echo " set to a random number. This can be used as a stable seed for tests to randomize behavior."
echo " Set this to a fixed value to get reproducible behavior."
+ echo " NFT_TEST_SHUFFLE_TESTS=*|n|y: control whether to randomly shuffle the order of tests. By default, if"
+ echo " tests are specified explicitly, they are not shuffled while they are shuffled when"
+ echo " all tests are run. The shuffling is based on NFT_TEST_RANDOM_SEED."
echo " TMPDIR=<PATH> : select a different base directory for the result data."
echo
echo " NFT_TEST_HAVE_<FEATURE>=*|y: Some tests requires certain features or will be skipped."
@@ -238,6 +242,7 @@ NFT_TEST_KEEP_LOGS="$(bool_y "$NFT_TEST_KEEP_LOGS")"
NFT_TEST_HAS_REALROOT="$NFT_TEST_HAS_REALROOT"
NFT_TEST_JOBS="${NFT_TEST_JOBS:-$_NFT_TEST_JOBS_DEFAULT}"
NFT_TEST_RANDOM_SEED="$NFT_TEST_RANDOM_SEED"
+NFT_TEST_SHUFFLE_TESTS="$NFT_TEST_SHUFFLE_TESTS"
NFT_TEST_SKIP_slow="$(bool_y "$NFT_TEST_SKIP_slow")"
DO_LIST_TESTS=
@@ -293,6 +298,9 @@ while [ $# -gt 0 ] ; do
;;
-s|--sequential)
NFT_TEST_JOBS=0
+ if [ -z "$NFT_TEST_SHUFFLE_TESTS" ] ; then
+ NFT_TEST_SHUFFLE_TESTS=n
+ fi
;;
-Q|--quick)
NFT_TEST_SKIP_slow=y
@@ -314,6 +322,9 @@ find_tests() {
if [ "${#TESTS[@]}" -eq 0 ] ; then
TESTS=( $(find_tests "$NFT_TEST_BASEDIR/testcases/") )
test "${#TESTS[@]}" -gt 0 || msg_error "Could not find tests"
+ if [ -z "$NFT_TEST_SHUFFLE_TESTS" ] ; then
+ NFT_TEST_SHUFFLE_TESTS=y
+ fi
fi
TESTSOLD=( "${TESTS[@]}" )
@@ -328,6 +339,8 @@ for t in "${TESTSOLD[@]}" ; do
fi
done
+NFT_TEST_SHUFFLE_TESTS="$(bool_y "$NFT_TEST_SHUFFLE_TESTS")"
+
if [ "$DO_LIST_TESTS" = y ] ; then
printf '%s\n' "${TESTS[@]}"
exit 0
@@ -496,6 +509,7 @@ msg_info "conf: NFT_TEST_HAS_UNSHARED_MOUNT=$(printf '%q' "$NFT_TEST_HAS_UNSHARE
msg_info "conf: NFT_TEST_KEEP_LOGS=$(printf '%q' "$NFT_TEST_KEEP_LOGS")"
msg_info "conf: NFT_TEST_JOBS=$NFT_TEST_JOBS"
msg_info "conf: NFT_TEST_RANDOM_SEED=$NFT_TEST_RANDOM_SEED"
+msg_info "conf: NFT_TEST_SHUFFLE_TESTS=$NFT_TEST_SHUFFLE_TESTS"
msg_info "conf: TMPDIR=$(printf '%q' "$_TMPDIR")"
echo
for KEY in $(compgen -v | grep '^NFT_TEST_SKIP_' | sort) ; do
@@ -709,6 +723,10 @@ job_wait()
done
}
+if [ "$NFT_TEST_SHUFFLE_TESTS" = y ] ; then
+ TESTS=( $(printf '%s\n' "${TESTS[@]}" | shuf --random-source=<("$NFT_TEST_BASEDIR/helpers/random-source.sh" "nft-test-shuffle-tests" "$NFT_TEST_RANDOM_SEED") ) )
+fi
+
TESTIDX=0
JOBS_N_RUNNING=0
for testfile in "${TESTS[@]}" ; do
--
2.41.0
