Add the ability to have those tests that fail return 0, and those
tests that succeed return 1. This is useful e.g. to run "--stress"
tests on tests that fail 99% of the time on some setup, i.e. to smoke
out the flaky run which yielded success.
In a subsequent commit a new SANITIZE=leak mode will make use of this.
Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@xxxxxxxxx>
---
t/t0000-basic.sh | 72 ++++++++++++++++++++++++++++++++++++++++++++++++
t/test-lib.sh | 45 ++++++++++++++++++++++++++++--
2 files changed, 115 insertions(+), 2 deletions(-)
diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh
index 17a268ccd1b..502b4bcf9ea 100755
--- a/t/t0000-basic.sh
+++ b/t/t0000-basic.sh
@@ -578,6 +578,78 @@ test_expect_success 'subtest: --run invalid range end' '
EOF_ERR
'
+test_expect_success 'subtest: --invert-exit-code without --immediate' '
+ run_sub_test_lib_test_err full-pass \
+ --invert-exit-code &&
+ check_sub_test_lib_test_err full-pass \
+ <<-\EOF_OUT 3<<-EOF_ERR
+ ok 1 - passing test #1
+ ok 2 - passing test #2
+ ok 3 - passing test #3
+ # passed all 3 test(s)
+ 1..3
+ # faking up non-zero exit with --invert-exit-code
+ EOF_OUT
+ EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code with --immediate: all passed' '
+ run_sub_test_lib_test_err full-pass \
+ --invert-exit-code --immediate &&
+ check_sub_test_lib_test_err full-pass \
+ <<-\EOF_OUT 3<<-EOF_ERR
+ ok 1 - passing test #1
+ ok 2 - passing test #2
+ ok 3 - passing test #3
+ # passed all 3 test(s)
+ 1..3
+ # faking up non-zero exit with --invert-exit-code
+ EOF_OUT
+ EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code without --immediate: partial pass' '
+ run_sub_test_lib_test partial-pass \
+ --invert-exit-code &&
+ check_sub_test_lib_test partial-pass <<-\EOF
+ ok 1 - passing test #1
+ not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+ # false
+ ok 3 - passing test #3
+ # failed 1 among 3 test(s)
+ 1..3
+ # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+ EOF
+'
+
+test_expect_success 'subtest: --invert-exit-code with --immediate: partial pass' '
+ run_sub_test_lib_test partial-pass \
+ --invert-exit-code --immediate &&
+ check_sub_test_lib_test partial-pass \
+ <<-\EOF_OUT 3<<-EOF_ERR
+ ok 1 - passing test #1
+ not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+ # false
+ 1..2
+ # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+ EOF_OUT
+ EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code --immediate: got a failure' '
+ run_sub_test_lib_test partial-pass \
+ --invert-exit-code --immediate &&
+ check_sub_test_lib_test_err partial-pass \
+ <<-\EOF_OUT 3<<-EOF_ERR
+ ok 1 - passing test #1
+ not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+ # false
+ 1..2
+ # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+ EOF_OUT
+ EOF_ERR
+'
+
test_expect_success 'subtest: tests respect prerequisites' '
write_and_run_sub_test_lib_test prereqs <<-\EOF &&
diff --git a/t/test-lib.sh b/t/test-lib.sh
index 118720493bb..31213b5f95b 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -238,6 +238,9 @@ parse_option () {
;;
esac
;;
+ --invert-exit-code)
+ invert_exit_code=t
+ ;;
*)
echo "error: unknown test option '$opt'" >&2; exit 1 ;;
esac
@@ -788,15 +791,31 @@ test_ok_ () {
finalize_test_case_output ok "$@"
}
+_invert_exit_code_failure_end_blurb () {
+ say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code"
+}
+
test_failure_ () {
failure_label=$1
test_failure=$(($test_failure + 1))
- say_color error "not ok $test_count - $1"
+ local pfx=""
+ if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE"
+ then
+ pfx="# TODO induced breakage (--invert-exit-code):"
+ fi
+ say_color error "not ok $test_count - ${pfx:+$pfx }$1"
shift
printf '%s\n' "$*" | sed -e 's/^/# /'
if test -n "$immediate"
then
say_color error "1..$test_count"
+ if test -n "$invert_exit_code"
+ then
+ finalize_test_output
+ _invert_exit_code_failure_end_blurb
+ GIT_EXIT_OK=t
+ exit 0
+ fi
_error_exit
fi
finalize_test_case_output failure "$failure_label" "$@"
@@ -1229,7 +1248,14 @@ test_done () {
esac
fi
- if test -z "$debug" && test -n "$remove_trash"
+ if test -n "$stress" && test -n "$invert_exit_code"
+ then
+ # We're about to move our "$TRASH_DIRECTORY"
+ # to "$TRASH_DIRECTORY.stress-failed" if
+ # --stress is combined with
+ # --invert-exit-code.
+ say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'"
+ elif test -z "$debug" && test -n "$remove_trash"
then
test -d "$TRASH_DIRECTORY" ||
error "Tests passed but trash directory already removed before test cleanup; aborting"
@@ -1242,6 +1268,14 @@ test_done () {
} ||
error "Tests passed but test cleanup failed; aborting"
fi
+
+ if test -z "$skip_all" && test -n "$invert_exit_code"
+ then
+ say_color warn "# faking up non-zero exit with --invert-exit-code"
+ GIT_EXIT_OK=t
+ exit 1
+ fi
+
test_at_end_hook_
GIT_EXIT_OK=t
@@ -1254,6 +1288,13 @@ test_done () {
say "1..$test_count"
fi
+ if test -n "$invert_exit_code"
+ then
+ _invert_exit_code_failure_end_blurb
+ GIT_EXIT_OK=t
+ exit 0
+ fi
+
GIT_EXIT_OK=t
exit 1 ;;
--
2.37.1.1064.gc96144cf387
