From: Prarit Bhargava <prarit@xxxxxxxxxx>
redhat/self-test: Provide better failure output
Currently when the tests fail there isn't a lot of information on which
test failed. This has resulted in confusion by developers.
Add a general bash function, check_status, that outputs some basic
information on which test failed and how to run the test to reproduce the
error.
Signed-off-by: Prarit Bhargava <prarit@xxxxxxxxxx>
diff --git a/redhat/self-test/0001-shellcheck.bats b/redhat/self-test/0001-shellcheck.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/0001-shellcheck.bats
+++ b/redhat/self-test/0001-shellcheck.bats
@@ -1,9 +1,12 @@
#!/usr/bin/env bats
# Purpose: This test runs shellcheck on all .sh files in the redhat directory.
+load test-lib.bash
+
@test "shellcheck" {
if ! test -x /usr/bin/shellcheck; then
skip "The ShellCheck package is not installed"
fi
- shellcheck $(find $BATS_TEST_DIRNAME/.. -name "*.sh" -not -path "$BATS_TEST_DIRNAME/../rpm/*")
+ run shellcheck $(find $BATS_TEST_DIRNAME/.. -name "*.sh" -not -path "$BATS_TEST_DIRNAME/../rpm/*")
+ check_status
}
diff --git a/redhat/self-test/1001-rpmlint.bats b/redhat/self-test/1001-rpmlint.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/1001-rpmlint.bats
+++ b/redhat/self-test/1001-rpmlint.bats
@@ -1,6 +1,8 @@
#!/usr/bin/env bats
# Purpose: This test runs rpmlint on the source rpm.
+load test-lib.bash
+
@test "rpmlint" {
if ! test -x /usr/bin/rpmlint; then
skip "The rpmlint package is not installed"
@@ -15,6 +17,5 @@
srpm=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm")
run rpmlint $srpm
- status=$?
- [ "$status" = 0 ]
+ check_result
}
diff --git a/redhat/self-test/1002-basic-structural-test.bats b/redhat/self-test/1002-basic-structural-test.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/1002-basic-structural-test.bats
+++ b/redhat/self-test/1002-basic-structural-test.bats
@@ -1,6 +1,12 @@
#!/usr/bin/env bats
# Purpose: This test runs tests on the SRPM.
+load test-lib.bash
+
+_SRPM_unpacks_OK() {
+ rpm2cpio "$srpm" | cpio -idm
+}
+
@test "SRPM unpacks OK" {
numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l)
if [ "$numsrpms" != "1" ]; then
@@ -14,9 +20,8 @@
fi
mkdir SRPMS
cd SRPMS
- rpm2cpio "$srpm" | cpio -idm
- status=$?
- [ "$status" = 0 ]
+ run _SRPM_unpacks_OK
+ check_status
popd >& /dev/null
}
@@ -30,7 +35,7 @@ numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l)
ls | wc
linuxname=$(ls linux*.tar.xz)
run tar --extract --xz -f "$linuxname"
- [ "$status" = 0 ]
+ check_status
popd >& /dev/null
}
@@ -44,7 +49,7 @@ numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l)
linuxtree=$(ls linux*.tar.xz)
linuxtree=${linuxtree/.tar.xz}
cd $linuxtree
- test -d arch && \
+ run test -d arch && \
test -d block && \
test -d certs && \
test -d crypto && \
@@ -66,7 +71,6 @@ numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l)
test -d tools && \
test -d usr && \
test -d virt
- status=$?
+ check_status
popd >& /dev/null
- [ "$status" = 0 ]
}
diff --git a/redhat/self-test/1003-rpminspect.bats b/redhat/self-test/1003-rpminspect.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/1003-rpminspect.bats
+++ b/redhat/self-test/1003-rpminspect.bats
@@ -1,6 +1,8 @@
#!/usr/bin/env bats
# Purpose: This test runs rpminspect on the SRPM.
+load test-lib.bash
+
@test "rpminspect" {
if ! test -x /usr/bin/rpminspect; then
skip "The rpminspect package is not installed"
@@ -15,5 +17,5 @@
srpm=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm")
run rpminspect $srpm
- [ "$status" = 0 ]
+ check_status
}
diff --git a/redhat/self-test/1005-dist-dump-variables.bats b/redhat/self-test/1005-dist-dump-variables.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/1005-dist-dump-variables.bats
+++ b/redhat/self-test/1005-dist-dump-variables.bats
@@ -3,6 +3,8 @@
# variables that are used in the specfile. This data is diff'd against a
# "known good" set of data and if there is a difference an error is reported.
+load test-lib.bash
+
@test "self-test-data check" {
mkdir -p $BATS_TMPDIR/data
RHDISTDATADIR=$BATS_TMPDIR/data make dist-self-test-data
@@ -10,6 +12,7 @@
redhat=$(make dist-dump-variables | grep "REDHAT=" | cut -d"=" -f2 | xargs)
echo "Diffing directories ${redhat}/self-test/data and $BATS_TMPDIR/data"
- diff -urNp -x create-data.sh ${redhat}/self-test/data $BATS_TMPDIR/data
+ run diff -urNp -x create-data.sh ${redhat}/self-test/data $BATS_TMPDIR/data
[ -d $BATS_TMPDIR ] && rm -rf $BATS_TMPDIR/data
+ check_status
}
diff --git a/redhat/self-test/1006-verify-SPEC-variables.bats b/redhat/self-test/1006-verify-SPEC-variables.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/1006-verify-SPEC-variables.bats
+++ b/redhat/self-test/1006-verify-SPEC-variables.bats
@@ -2,8 +2,9 @@
# Purpose: This test looks at the spec file variable replacement code in
# redhat/genspec.sh and confirms that each variable begins with "SPEC".
-@test "verify SPEC variables" {
+load test-lib.bash
+_verify_SPEC_variables() {
# This looks at the code and replaces each / with a new-line character, removes
# any whitespace and entry entries beginning with valid "%%SPEC" or $"SPEC".
# "$SOURCES" lines are also okay as it is used to point to the changelog and
@@ -23,3 +24,8 @@ do
esac
done
}
+
+@test "verify SPEC variables" {
+ run _verify_SPEC_variables
+ check_status
+}
diff --git a/redhat/self-test/2001-dist-release.bats b/redhat/self-test/2001-dist-release.bats
index blahblah..blahblah 100644
--- a/redhat/self-test/2001-dist-release.bats
+++ b/redhat/self-test/2001-dist-release.bats
@@ -2,7 +2,9 @@
# Purpose: These are general dist-release tests. They are run from a git
# worktree created by the first test.
-@test "dist-release prologue" {
+load test-lib.bash
+
+@test "dist-release setup worktree" {
git worktree add $BATS_TMPDIR/distrelease
cd $BATS_TMPDIR/distrelease
# All the tests start off with 'make dist-release', so we can pull
@@ -21,7 +23,12 @@
logb=($(git log --oneline -n 2 | tail -1))
# If SHA1 in loga is the same as the SHA1 in logb, then no
# 2nd commit has been created and the test has succeeded:
- [ ${loga[0]} = ${logb[0]} ]
+ run [ ${loga[0]} = ${logb[0]} ]
+ check_status
+}
+
+_dist-release_test_2() {
+ echo $pkgrelease | grep -q -w "$build"
}
@test "dist-release test 2" {
@@ -39,11 +46,17 @@
((build--))
echo "pkgrelease=$pkgrelease"
echo "build=$build"
- echo $pkgrelease | grep -q -w "$build"
- status=$?
- [ "$status" = 0 ]
+ run _dist-release_test_2
+ check_status
}
+_dist-release_test_3() {
+ [ "$changelog" = "$gitlog" ]
+}
+
+# Note, when running this test on the command line you may have to specifiy the
+# RHEL_MAJOR and RHEL_MINOR variables, for example,
+# RHEL_MAJOR=9 RHEL_MINOR=99 bats redhat/self-test/2001-dist-release.bats
@test "dist-release test 3" {
# Test whether the version in the commit message matches
# the version in the change log.
@@ -56,11 +69,12 @@
gitlog=${commit##*\[redhat\] }
# This time, strip off "kernel-" also:
gitlog=${gitlog/kernel-/}
- echo "The kernel version in the changelog ($changelog) differs from the version in the git log ($gitlog)"
- [ "$changelog" = "$gitlog" ]
+ echo "The kernel version in the changelog-${RHEL_MAJOR}.${RHEL_MINOR} ("${changelog}") differs from the version in the git log ($gitlog)"
+ run _dist-release_test_3
+ check_status
}
-@test "dist-release epilogue" {
+@test "dist-release cleanup worktree" {
git worktree remove --force $BATS_TMPDIR/distrelease
git branch -D distrelease
}
diff --git a/redhat/self-test/3001-Makefile-contents.bats b/redhat/self-test/3001-Makefile-contents.bats
index blahblah..blahblah 100755
--- a/redhat/self-test/3001-Makefile-contents.bats
+++ b/redhat/self-test/3001-Makefile-contents.bats
@@ -2,18 +2,28 @@
# Purpose: This is a test that verifies that Makefile.variable variable
# declarations are all declared with "?="
-@test "Makefile variable declarations" {
- # By design, only the Makefile.variables file should have ?= declarations
+load test-lib.bash
+
+_Makefile_variable_declarations_1() {
+ git grep "?=" $BATS_TEST_DIRNAME/../Makefile.variables | wc -l
+}
- value=$(git grep "?=" Makefile.variables | wc -l)
- if [ $value -eq 0 ]; then
+_Makefile_variable_declarations_2() {
+ git grep "?=" $BATS_TEST_DIRNAME/../Makefile | grep -v "\"?=" | wc -l
+}
+
+@test "Makefile variable declarations" {
+ run _Makefile_variable_declarations_1
+ if [ "$output" -eq 0 ]; then
echo "Test failed: No ?= variables found in Makefile.variables"
- exit 1
+ status=1
fi
+ check_status
- value=$(git grep "?=" Makefile | grep -v "\"?=" | wc -l)
- if [ $value -gt 0 ]; then
+ run _Makefile_variable_declarations_2
+ if [ "$output" -ne 0 ]; then
echo "Test failed: Makefile should not ?= declarations."
- exit 1
+ status=1
fi
+ check_status
}
diff --git a/redhat/self-test/test-lib.bash b/redhat/self-test/test-lib.bash
new file mode 100644
index blahblah..blahblah 100644
--- /dev/null
+++ b/redhat/self-test/test-lib.bash
@@ -0,0 +1,16 @@
+#!/usr/bin/bash
+
+# This function makes use of bats built-in run function and its status and output variables.
+check_status() {
+ if [ "$status" -eq 0 ]; then
+ return 0
+ fi
+
+ # report the error
+ echo "$output"
+ echo "------------------"
+ expath=$(echo "$BATS_TEST_FILENAME" | rev | cut -d'/' -f-3 | rev)
+ echo -n "This redhat/self-test test has failed. You can run all tests by executing 'make dist-self-test', or just this test by executing 'bats $expath'."
+ exit 1
+}
+
--
https://gitlab.com/cki-project/kernel-ark/-/merge_requests/1881
_______________________________________________
kernel mailing list -- kernel@xxxxxxxxxxxxxxxxxxxxxxx
To unsubscribe send an email to kernel-leave@xxxxxxxxxxxxxxxxxxxxxxx
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedoraproject.org/archives/list/kernel@xxxxxxxxxxxxxxxxxxxxxxx
Do not reply to spam on the list, report it: https://pagure.io/fedora-infrastructure
