Add known issue support so it's easier to know if a failure is a known
issue or a new one (and potential regression), and mark the whole test
as failure (check returns non-zero) only when there're unknown failures.
A new $KNOWNISSUE_DIR dir is introduced (default to `pwd`/knownissue),
and common known issues across filesystems/arches/configs can be listed
in $KNOWNISSUE_DIR/generic and $KNOWNISSUE_DIR/$FSTYP, they are the
default known issue files.
When testing with section in config, an additional section-specific
known issue file is processed, $KNOWNISSUE_DIR/$section_name, all known
failures for this specific config can be listed here.
A new "-k" option to check is added, it's used to specify user-provided
known issue file rather than the default files. When "-k" is used, only
the known issues in this file are processed, $KNOWNISSUE_DIR/{generic,
$FSTYP, $section_name} are ignored.
The format of the known issue file is:
# comments are starting with '#'
# first column is test name
# second column indicates if the test should be skipped, yes to skip
# the third column is the note about this failure
# test name skip? comments
generic/001 no triggers WARNING ...
generic/002 yes crash kernel, skip, patch xxx should fix it
...
After test, test summary is printed, e.g.
Ran: ext4/005 generic/001 generic/018
Failures: ext4/005 generic/018
Failed 2 of 3 tests
Known failures: ext4/005
1 of 2 failed tests are known failures
Unknown failures: generic/018
1 of 2 failed tests are unknown failures
Examples:
# run specific section with user provided known issue file
./check -g auto -s ext4_1k_data_journal -k /path/to/known_issue_file
Signed-off-by: Eryu Guan <eguan@xxxxxxxxxx>
---
check | 179 ++++++++++++++++++++++++++++++++++++++++------------------
common/config | 4 ++
2 files changed, 127 insertions(+), 56 deletions(-)
diff --git a/check b/check
index 135a9fb..dc91580 100755
--- a/check
+++ b/check
@@ -29,6 +29,10 @@ n_bad=0
sum_bad=0
bad=""
notrun=""
+n_knownbad=0
+knownbad=""
+n_unknownbad=0
+unknownbad=""
interrupt=true
diff="diff -u"
showme=false
@@ -73,6 +77,7 @@ check options
-d dump test output to stdout
--large-fs optimise scratch device for large filesystems
-s section run only specified section from config file
+ -k file use this known issue file other than the default one
testlist options
-g group[,group...] include tests from these groups
@@ -134,6 +139,34 @@ trim_test_list()
mv $tmp.tmp $tmp.list
}
+# save tests in known issue file that should be skipped in $tmp.xlist
+get_skip_test_list()
+{
+ $AWK_PROG '{if ($2 == "yes") print $1}' $1 | grep -v "^#" >>$tmp.xlist
+}
+
+# save all tests in known issue file in $tmp.knownissue
+get_knownissue_list()
+{
+ $AWK_PROG '{print $1}' $1 | grep -v "^#" >>$tmp.knownissue
+}
+
+# check if the given test is a known issue
+is_knownissue()
+{
+ grep -qF $1 $tmp.knownissue 2>/dev/null
+}
+
+process_knownissue()
+{
+ for file in $*; do
+ if [ ! -s $file ]; then
+ continue
+ fi
+ get_knownissue_list $file
+ get_skip_test_list $file
+ done
+}
_wallclock()
{
@@ -228,6 +261,7 @@ while [ $# -gt 0 ]; do
fi
;;
-s) RUN_SECTION="$RUN_SECTION $2"; shift ;;
+ -k) KNOWNISSUE_FILE="$2"; shift ;;
-l) diff="diff" ;;
-udiff) diff="$diff -u" ;;
@@ -262,6 +296,17 @@ if ! . ./common/config; then
exit 1
fi
+# process known issue file to
+# 1. get all known issues
+# 2. get tests need to be skipped from known issue list
+if [ -n "$KNOWNISSUE_FILE" ]; then
+ # have user specified known issue file
+ process_knownissue $KNOWNISSUE_FILE
+else
+ # no user specified known issue file, use the default files
+ process_knownissue $KNOWNISSUE_DIR/generic $KNOWNISSUE_BASE/$FSTYP
+fi
+
# Process tests from command line now.
if $have_test_arg; then
while [ $# -gt 0 ]; do
@@ -304,77 +349,88 @@ fi
_wipe_counters()
{
- n_try="0"
- n_bad="0"
- unset try notrun bad
+ n_try=0
+ n_bad=0
+ n_knownbad=0
+ n_unknownbad=0
+ unset try notrun bad knownbad unknownbad
+}
+
+_log()
+{
+ echo -e "$@"
+ echo -e "$@" | fmt >>$check.log
+ echo -e "$@" >>$tmp.summary
}
_wrapup()
{
- seq="check"
- check="$RESULT_BASE/check"
-
- if $showme
- then
- :
- elif $needwrap
- then
- if [ -f $check.time -a -f $tmp.time ]
- then
- cat $check.time $tmp.time \
- | $AWK_PROG '
+ seq="check"
+ check="$RESULT_BASE/check"
+
+ if $showme; then
+ :
+ elif $needwrap; then
+ if [ -f $check.time -a -f $tmp.time ]; then
+ cat $check.time $tmp.time | $AWK_PROG '
{ t[$1] = $2 }
END { if (NR > 0) {
for (i in t) print i " " t[i]
}
}' \
- | sort -n >$tmp.out
- mv $tmp.out $check.time
- fi
+ | sort -n >$tmp.out
+ mv $tmp.out $check.time
+ fi
- echo "" >>$check.log
- date >>$check.log
- echo $list | fmt | sed -e 's/^/ /' -e "s;$SRC_DIR/;;g" >>$check.log
- $interrupt && echo "Interrupted!" >>$check.log
+ echo "" >>$check.log
+ date >>$check.log
+ echo $list | fmt | sed -e 's/^/ /' -e "s;$SRC_DIR/;;g" >>$check.log
+ $interrupt && echo "Interrupted!" >>$check.log
- echo "SECTION -- $section" >>$tmp.summary
- echo "=========================" >>$tmp.summary
- if [ ! -z "$n_try" -a $n_try != 0 ]
- then
- echo "Ran:$try"
- echo "Ran:$try" >>$tmp.summary
- fi
+ echo "SECTION -- $section" >>$tmp.summary
+ echo "=========================" >>$tmp.summary
+ if [ ! -z "$n_try" -a $n_try != 0 ]; then
+ echo "Ran:$try"
+ echo "Ran:$try" >>$tmp.summary
+ fi
- if [ ! -z "$notrun" ]
- then
- echo "Not run:$notrun"
- echo "Not run:$notrun" >>$check.log
- echo "Not run:$notrun" >>$tmp.summary
+ if [ ! -z "$notrun" ]; then
+ _log "Not run:$notrun"
+ fi
+
+ if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
+ for t in $bad; do
+ if is_knownissue $t; then
+ knownbad="$knownbad $t"
+ n_knownbad=`expr $n_knownbad + 1`
+ else
+ unknownbad="$unknownbad $t"
+ n_unknownbad=`expr $n_unknownbad + 1`
+ fi
+ done
+ _log "Failures:$bad"
+ _log "Failed $n_bad of $n_try tests"
+ if [ ! -z "$n_knownbad" -a $n_knownbad != 0 ]; then
+ _log "Known failures:$knownbad"
+ _log "$n_knownbad of $n_bad failed tests are known failures"
+ fi
+ if [ ! -z "$n_unknownbad" -a $n_unknownbad != 0 ]; then
+ _log "Unknown failures:$unknownbad"
+ _log "$n_unknownbad of $n_bad failed tests are unknown failures"
+ fi
+ else
+ _log "Passed all $n_try tests"
+ fi
+ echo "" >>$tmp.summary
+ needwrap=false
fi
- if [ ! -z "$n_bad" -a $n_bad != 0 ]
- then
- echo "Failures:$bad"
- echo "Failed $n_bad of $n_try tests"
- echo "Failures:$bad" | fmt >>$check.log
- echo "Failed $n_bad of $n_try tests" >>$check.log
- echo "Failures:$bad" >>$tmp.summary
- echo "Failed $n_bad of $n_try tests" >>$tmp.summary
- else
- echo "Passed all $n_try tests"
- echo "Passed all $n_try tests" >>$check.log
- echo "Passed all $n_try tests" >>$tmp.summary
+ sum_bad=`expr $sum_bad + $n_unknownbad`
+ _wipe_counters
+ rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
+ if ! $OPTIONS_HAVE_SECTIONS; then
+ rm -f $tmp.*
fi
- echo "" >>$tmp.summary
- needwrap=false
- fi
-
- sum_bad=`expr $sum_bad + $n_bad`
- _wipe_counters
- rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
- if ! $OPTIONS_HAVE_SECTIONS; then
- rm -f $tmp.*
- fi
}
_summary()
@@ -434,9 +490,20 @@ for section in $HOST_OPTIONS_SECTIONS; do
status=1
exit
fi
+ mkdir -p $KNOWNISSUE_DIR
+ if [ ! -d $KNOWNISSUE_DIR ]; then
+ echo "failed to create known issue directory $KNOWNISSUE_DIR"
+ status=1
+ exit
+ fi
if $OPTIONS_HAVE_SECTIONS; then
echo "SECTION -- $section"
+ # process section-specific known issue file
+ # if no known issue file is specified in command line
+ if [ -z "$KNOWNISSUE_FILE" ]; then
+ process_knownissue $KNOWNISSUE_DIR/$section
+ fi
fi
if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
diff --git a/common/config b/common/config
index 079e831..ff2b37a 100644
--- a/common/config
+++ b/common/config
@@ -485,6 +485,10 @@ get_next_config() {
if [ -z "$RESULT_BASE" ]; then
export RESULT_BASE="$here/results/"
fi
+ # set default KNOWNISSUE_DIR
+ if [ -z "$KNOWNISSUE_DIR" ]; then
+ export KNOWNISSUE_DIR="$here/knownissue"
+ fi
# Mandatory Config values.
MC=""
--
2.5.0
--
To unsubscribe from this list: send the line "unsubscribe fstests" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
