summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ChangeLog41
-rw-r--r--lib/Automake/tests/Makefile.in16
-rw-r--r--lib/am/check.am41
-rwxr-xr-xlib/test-driver16
-rw-r--r--tests/Makefile.am11
-rw-r--r--tests/Makefile.in29
-rwxr-xr-xtests/parallel-tests-empty-testlogs.test86
-rwxr-xr-xtests/parallel-tests2.test43
-rwxr-xr-xtests/parallel-tests7.test26
-rwxr-xr-xtests/test-driver-custom-html.test104
-rwxr-xr-xtests/test-driver-custom-multitest-recheck.test223
-rwxr-xr-xtests/test-driver-custom-multitest-recheck2.test172
-rwxr-xr-xtests/test-driver-custom-multitest.test191
-rwxr-xr-xtests/test-driver-custom-no-extra-driver.test6
-rwxr-xr-xtests/test-driver-custom-no-html.test67
-rwxr-xr-xtests/test-driver-custom-xfail-tests.test27
-rw-r--r--tests/trivial-test-driver101
17 files changed, 1142 insertions, 58 deletions
diff --git a/ChangeLog b/ChangeLog
index c008877b2..d6dbd1341 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,46 @@
2011-06-21 Stefano Lattarini <stefano.lattarini@gmail.com>
+ parallel-tests: allow each test to have multiple results
+ With this change, we improve the code creating the `test-suite.log'
+ global log and the console testsuite summary to make it able to
+ grasp multiple results per test script. This is required in order
+ to introduce the planned support for test protocols, like TAP and
+ SubUnit, which can indeed run multiple tests per test script, each
+ with its individual result.
+ The implementation makes use of a custom reStructuredText field
+ `:test-result:'.
+ Note that no new documentation is added by this change; that is
+ be left for follow-up changes.
+ * lib/check.am ($(TEST_SUITE_LOG)): When processing .log files,
+ recognize a report of a test's result only if it is declared with
+ the custom `:test-result:' reStructuredText field placed at the
+ beginning of a line. Extend and add explanatory comments.
+ (recheck, recheck-html): Add explanatory comments.
+ * lib/test-driver: Write an appropriate reStructuredText field
+ `:test-result:' in the generated log file. Use a reStructuredText
+ transition to better separate the test outcome report from the
+ test script's registered output. Improve comments.
+ * tests/test-driver-custom-xfail-tests.test: Adjust.
+ * tests/parallel-tests7.test: Adjust.
+ * tests/parallel-tests-empty-testlogs.test: New test.
+ * tests/parallel-tests-recheck-override.test: Likewise.
+ * tests/parallel-tests2.test: Extend and keep more in-sync with ...
+ * tests/test-driver-custom-html.test: ... this new related test.
+ * tests/test-driver-custom-no-html.test: New test.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/trivial-test-driver: New file, used by the last four tests
+ above.
+ * tests/Makefile.am (TESTS): Update.
+ (EXTRA_DIST): Distribute `trivial-test-driver'.
+ (test-driver-custom-multitest.log): Depend on `trivial-test-driver'.
+ (test-driver-custom-multitest-recheck.log): Likewise.
+ (test-driver-custom-multitest-recheck2.log): Likewise.
+ (test-driver-custom-html.log): Likewise.
+
+2011-06-21 Stefano Lattarini <stefano.lattarini@gmail.com>
+
parallel-tests: allow custom driver scripts
Allow suffix-based definition of custom "driver script" for the
test scripts. These driver scripts will be responsible of
diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in
index ed1ddbe46..9c344f539 100644
--- a/lib/Automake/tests/Makefile.in
+++ b/lib/Automake/tests/Makefile.in
@@ -343,12 +343,16 @@ cscope cscopelist:
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- list='$(TEST_LOGS)'; \
- results=`for f in $$list; do \
- test -r $$f && read line < $$f && echo "$$line" \
- || echo FAIL; \
- done`; \
+ @$(am__sh_e_setup); \
+ rst_magic=":test-result:"; \
+ list='$(TEST_LOGS)'; \
+ list2=`for f in $$list; do test ! -r $$f || echo $$f; done`; \
+ results1=`for f in $$list; do test -r $$f || echo FAIL; done`; \
+ results2=''; \
+ if test -n "$$list2"; then \
+ results2=`sed -n "s/^$$rst_magic[ ]*//p" $$list2`; \
+ fi; \
+ results=`echo "$$results1" && echo "$$results2"` \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
diff --git a/lib/am/check.am b/lib/am/check.am
index 6ddf61f4c..062c59c52 100644
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -139,12 +139,25 @@ esac; \
$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- list='$(TEST_LOGS)'; \
- results=`for f in $$list; do \
- test -r $$f && read line < $$f && echo "$$line" \
- || echo FAIL; \
- done`; \
+ @$(am__sh_e_setup); \
+## The custom reStructuredText field used to register the outcome of
+## a test. This is for supporting test protocols, such as TAP and
+## SubUnit, where a single test script can run multiple tests, each
+## with its own outcome.
+ rst_magic=":test-result:"; \
+## All test logs.
+ list='$(TEST_LOGS)'; \
+## Readable test logs.
+ list2=`for f in $$list; do test ! -r $$f || echo $$f; done`; \
+## Each unreadable test log counts as a failed test.
+ results1=`for f in $$list; do test -r $$f || echo FAIL; done`; \
+## Extract the outcome of all the testcases from the test logs.
+ results2=''; \
+ if test -n "$$list2"; then \
+ results2=`sed -n "s/^$$rst_magic[ ]*//p" $$list2`; \
+ fi; \
+## Prepare the test suite summary.
+ results=`echo "$$results1" && echo "$$results2"` \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
@@ -184,6 +197,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
msg="$$msg($$skip tests were not run). "; \
fi; \
fi; \
+## Write "global" testsuite log.
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
@@ -191,6 +205,16 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
+## Here we assume that the test driver writes a proper summary for the
+## test script on the first line. Requiring this is not a limitation,
+## but a feature, since this way a custom test driver is allowed to decide
+## what the outcome is in case of conflicting testcase results in a test
+## script. For example, if a test script reports 8 successful testcases
+## and 2 skipped testcases, some drivers might report that globally as a
+## SKIP, while others as a PASS.
+## FIXME: This should be documented in the automake manual. The above
+## FIXME: explanation is indeed more appropriate for the manual than for
+## FIXME: comments in code.
for f in $$list; do \
test -r $$f && read line < $$f || line=; \
case $$line in \
@@ -207,6 +231,7 @@ $(TEST_SUITE_LOG): $(TEST_LOGS)
fi; \
fi; \
test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
+## Emit the test summary on the console, and exit.
$(am__tty_colors); \
if $$exit; then \
col="$$grn"; \
@@ -288,6 +313,10 @@ recheck recheck-html:
list=`for f in $$list; do \
test -f $$f || continue; \
if test -r $$f && read line < $$f; then \
+## Here we assume that the test driver writes a proper summary for the
+## test script on the first line. See the comments in the rules of
+## $(TEST_SUITE_LOG) above for why we consider this acceptable and even
+## advisable.
case $$line in FAIL*|XPASS*) echo $$f;; esac; \
else echo $$f; fi; \
done | tr '\012\015' ' '`; \
diff --git a/lib/test-driver b/lib/test-driver
index 87533240c..31640c3f2 100755
--- a/lib/test-driver
+++ b/lib/test-driver
@@ -113,9 +113,21 @@ case $estatus:$expect_failure in
*:yes) col=$lgn; res=XFAIL;;
*:*) col=$red; res=FAIL ;;
esac
+
+# Report outcome to console.
echo "${col}${res}${std}: $test_name"
-echo "$res: $test_name (exit: $estatus)" | rst_section > $logfile
-cat $tmpfile >> $logfile
+
+# Now write log file.
+{
+ echo "$res: $test_name (exit: $estatus)" | rst_section
+ echo ":test-result: $res (exit status: $estatus)"
+ # Use a reStructuredText transition to better separate the test
+ # outcome report from its registered output.
+ echo
+ echo ------------
+ echo
+ cat $tmpfile
+} > $logfile
rm -f $tmpfile
# Local Variables:
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 86dc8214e..f1ccb7b21 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -723,10 +723,16 @@ parallel-tests-unreadable-log.test \
parallel-tests-subdir.test \
parallel-tests-interrupt.test \
parallel-tests-reset-term.test \
+parallel-tests-empty-testlogs.test \
parallel-test-driver-install.test \
test-driver-custom-no-extra-driver.test \
test-driver-custom.test \
test-driver-custom-xfail-tests.test \
+test-driver-custom-multitest.test \
+test-driver-custom-multitest-recheck.test \
+test-driver-custom-multitest-recheck2.test \
+test-driver-custom-html.test \
+test-driver-custom-no-html.test \
test-driver-fail.test \
parse.test \
percent.test \
@@ -1070,6 +1076,11 @@ $(parallel_tests)
EXTRA_DIST += $(TESTS)
+test-driver-custom-multitest.log: trivial-test-driver
+test-driver-custom-multitest-recheck.log: trivial-test-driver
+test-driver-custom-multitest-recheck2.log: trivial-test-driver
+test-driver-custom-html.log: trivial-test-driver
+EXTRA_DIST += trivial-test-driver
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 902079563..f06562b45 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -282,7 +282,7 @@ top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
MAINTAINERCLEANFILES = $(parallel_tests) $(instspc_tests)
EXTRA_DIST = ChangeLog-old gen-parallel-tests instspc-tests.sh \
- $(TESTS)
+ $(TESTS) trivial-test-driver
XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \
override-conditional-2.test pr8365-remake-timing.test \
yacc-dist-nobuild-subdir.test txinfo5.test \
@@ -983,10 +983,16 @@ parallel-tests-unreadable-log.test \
parallel-tests-subdir.test \
parallel-tests-interrupt.test \
parallel-tests-reset-term.test \
+parallel-tests-empty-testlogs.test \
parallel-test-driver-install.test \
test-driver-custom-no-extra-driver.test \
test-driver-custom.test \
test-driver-custom-xfail-tests.test \
+test-driver-custom-multitest.test \
+test-driver-custom-multitest-recheck.test \
+test-driver-custom-multitest-recheck2.test \
+test-driver-custom-html.test \
+test-driver-custom-no-html.test \
test-driver-fail.test \
parse.test \
percent.test \
@@ -1378,12 +1384,16 @@ cscope cscopelist:
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- list='$(TEST_LOGS)'; \
- results=`for f in $$list; do \
- test -r $$f && read line < $$f && echo "$$line" \
- || echo FAIL; \
- done`; \
+ @$(am__sh_e_setup); \
+ rst_magic=":test-result:"; \
+ list='$(TEST_LOGS)'; \
+ list2=`for f in $$list; do test ! -r $$f || echo $$f; done`; \
+ results1=`for f in $$list; do test -r $$f || echo FAIL; done`; \
+ results2=''; \
+ if test -n "$$list2"; then \
+ results2=`sed -n "s/^$$rst_magic[ ]*//p" $$list2`; \
+ fi; \
+ results=`echo "$$results1" && echo "$$results2"` \
all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[ ]*//'`; \
fail=`echo "$$results" | grep -c '^FAIL'`; \
pass=`echo "$$results" | grep -c '^PASS'`; \
@@ -1735,6 +1745,11 @@ $(instspc_tests): Makefile.am
instspc-data.log: instspc-tests.sh
$(instspc_tests:.test=.log): instspc-tests.sh instspc-data.log
+test-driver-custom-multitest.log: trivial-test-driver
+test-driver-custom-multitest-recheck.log: trivial-test-driver
+test-driver-custom-multitest-recheck2.log: trivial-test-driver
+test-driver-custom-html.log: trivial-test-driver
+
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
diff --git a/tests/parallel-tests-empty-testlogs.test b/tests/parallel-tests-empty-testlogs.test
new file mode 100755
index 000000000..593dce315
--- /dev/null
+++ b/tests/parallel-tests-empty-testlogs.test
@@ -0,0 +1,86 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - empty TESTS
+# - empty TEST_LOGS
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_CONFIG_FILES([sub1/Makefile sub2/Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+SUBDIRS = sub1 sub2
+END
+
+mkdir sub1 sub2
+
+cat > sub1/Makefile.am << 'END'
+TESTS =
+check-local:
+ echo $(TEST_LOGS) | grep . && exit 1; exit 0
+END
+
+cat > sub2/Makefile.am << 'END'
+TESTS = foo.test
+END
+
+cat > sub2/foo.test <<'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x sub2/foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+no_test_has_run ()
+{
+ ls -1 *.log | grep -v '^test-suite\.log$' | grep . && Exit 1
+ grep ' 0 tests passed' test-suite.log
+ :
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+ $srcdir/configure
+ cd sub1
+ VERBOSE=yes $MAKE check
+ no_test_has_run
+ cd ../sub2
+ VERBOSE=yes TESTS='' $MAKE -e check
+ no_test_has_run
+ VERBOSE=yes TEST_LOGS='' $MAKE -e check
+ no_test_has_run
+ cd ..
+ $MAKE check
+ cat sub2/foo.log
+ $MAKE distclean
+ cd $srcdir
+done
+
+:
diff --git a/tests/parallel-tests2.test b/tests/parallel-tests2.test
index 8fe5d3027..ab390f862 100755
--- a/tests/parallel-tests2.test
+++ b/tests/parallel-tests2.test
@@ -15,8 +15,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Check parallel-tests features:
-# - check-html
-# - recheck-html
+# - check-html
+# - recheck-html
+# Keep this in sync with sister test `test-driver-custom-html.test'.
parallel_tests=yes
required=rst2html
@@ -35,22 +36,25 @@ bla:
CLEANFILES = bla
END
-cat >>foo.test <<'END'
+cat > foo.test <<'END'
#! /bin/sh
echo "this is $0"
test -f bla || exit 1
exit 0
END
-cat >>bar.test <<'END'
+
+cat > bar.test <<'END'
#! /bin/sh
echo "this is $0"
exit 99
END
-cat >>baz.test <<'END'
+
+cat > baz.test <<'END'
#! /bin/sh
echo "this is $0"
exit 1
END
+
chmod a+x foo.test bar.test baz.test
$ACLOCAL
@@ -58,30 +62,39 @@ $AUTOCONF
$AUTOMAKE -a
./configure
-$MAKE check-html >stdout && { cat stdout; Exit 1; }
-cat stdout
+
+$MAKE check-html && Exit 1
test -f mylog.html
+# check-html should cause check_SCRIPTS to be created.
+test -f bla
+
+# "make clean" should remove HTML files.
+$MAKE clean
+test ! -f mylog.html
+test ! -f bla
# Always create the HTML output, even if there were no failures.
rm -f mylog.html
-env TESTS=foo.test $MAKE -e check-html >stdout || { cat stdout; Exit 1; }
-cat stdout
+env TESTS=foo.test $MAKE -e check-html
test -f mylog.html
-# Create HTML output also with recheck-html
+# Create summarizing HTML output also with recheck-html.
rm -f mylog.html
-env TESTS=foo.test $MAKE -e recheck-html >stdout || { cat stdout; Exit 1; }
-cat stdout
+env TESTS=foo.test $MAKE -e recheck-html
test -f mylog.html
-# check-html and recheck-html should cause check_SCRIPTS to be created,
-# and recheck-html should rerun no tests if check has not been run.
+# check-html should cause check_SCRIPTS to be created.
$MAKE clean
-env TESTS=foo.test $MAKE -e check-html
+env TEST_LOGS=foo.log $MAKE -e check-html
test -f bla
+test -f foo.log
+test -f mylog.html
+# recheck-html should cause check_SCRIPTS to be created, and should rerun
+# no tests if it appears that check has not been run.
$MAKE clean
env TESTS=foo.test $MAKE -e recheck-html
test -f bla
test ! -f foo.log
test -f mylog.html
+
:
diff --git a/tests/parallel-tests7.test b/tests/parallel-tests7.test
index c29d32c21..40fc4a5bf 100755
--- a/tests/parallel-tests7.test
+++ b/tests/parallel-tests7.test
@@ -30,9 +30,9 @@ cat > Makefile.am << 'END'
TESTS = foo.chk bar.test $(check_PROGRAMS) sub/test
check_PROGRAMS = baz bla.test bli.suff
TEST_EXTENSIONS = .chk .test
-CHK_LOG_COMPILER = ./chk-driver
-TEST_LOG_COMPILER = ./test-driver
-LOG_COMPILER = ./noext-driver
+CHK_LOG_COMPILER = ./chk-compiler
+TEST_LOG_COMPILER = ./test-compiler
+LOG_COMPILER = ./noext-compiler
AM_CHK_LOG_FLAGS = 1
CHK_LOG_FLAGS = 2
AM_TEST_LOG_FLAGS = 3
@@ -43,7 +43,7 @@ END
mkdir sub
-cat >chk-driver <<'END'
+cat >chk-compiler <<'END'
#! /bin/sh
echo $0 "$@"
shift
@@ -51,9 +51,9 @@ shift
exec "$@"
exit 127
END
-chmod a+x chk-driver
-cp chk-driver test-driver
-cp chk-driver noext-driver
+chmod a+x chk-compiler
+cp chk-compiler test-compiler
+cp chk-compiler noext-compiler
cat >foo.chk << 'END'
#! /bin/sh
@@ -79,10 +79,10 @@ $AUTOMAKE -a
./configure
$MAKE
$MAKE check
-grep 'chk-driver *1 *2' foo.log
-grep 'test-driver *3 *4' bar.log
-grep 'noext-driver *5 *6' baz.log
-grep 'test-driver *3 *4' bla.log
-grep 'noext-driver *5 *6' bli.suff.log
-grep 'noext-driver *5 *6' sub/test.log
+grep 'chk-compiler *1 *2' foo.log
+grep 'test-compiler *3 *4' bar.log
+grep 'noext-compiler *5 *6' baz.log
+grep 'test-compiler *3 *4' bla.log
+grep 'noext-compiler *5 *6' bli.suff.log
+grep 'noext-compiler *5 *6' sub/test.log
:
diff --git a/tests/test-driver-custom-html.test b/tests/test-driver-custom-html.test
new file mode 100755
index 000000000..5663ee6b2
--- /dev/null
+++ b/tests/test-driver-custom-html.test
@@ -0,0 +1,104 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers features:
+# - check-html
+# - recheck-html
+# Keep this in sync with sister test `parallel-tests2.test'.
+
+parallel_tests=yes
+required=rst2html
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) ./trivial-test-driver
+TEST_SUITE_LOG = mylog.log
+TESTS = foo.test bar.test baz.test
+check_SCRIPTS = bla
+bla:
+ echo bla > $@
+CLEANFILES = bla
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+if test -f bla; the
+ echo "PASS: this is $0"
+else
+ echo "FAIL: this is $0"
+fi
+END
+
+cat > bar.test <<'END'
+#! /bin/sh
+echo "FAIL: this is $0"
+END
+
+cat > baz.test <<'END'
+#! /bin/sh
+echo "FAIL: this is $0"
+END
+
+chmod a+x foo.test bar.test baz.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check-html && Exit 1
+test -f mylog.html
+# check-html should cause check_SCRIPTS to be created.
+test -f bla
+
+# "make clean" should remove HTML files.
+$MAKE clean
+test ! -f mylog.html
+test ! -f bla
+
+# Always create the HTML output, even if there were no failures.
+rm -f mylog.html
+env TESTS=foo.test $MAKE -e check-html
+test -f mylog.html
+
+# Create summarizing HTML output also with recheck-html.
+rm -f mylog.html
+env TESTS=foo.test $MAKE -e recheck-html
+test -f mylog.html
+
+# check-html should cause check_SCRIPTS to be created.
+$MAKE clean
+env TEST_LOGS=foo.log $MAKE -e check-html
+test -f bla
+test -f foo.log
+test -f mylog.html
+# recheck-html should cause check_SCRIPTS to be created, and should rerun
+# no tests if it appears that check has not been run.
+$MAKE clean
+env TESTS=foo.test $MAKE -e recheck-html
+test -f bla
+test ! -f foo.log
+test -f mylog.html
+
+:
diff --git a/tests/test-driver-custom-multitest-recheck.test b/tests/test-driver-custom-multitest-recheck.test
new file mode 100755
index 000000000..e0f010f1d
--- /dev/null
+++ b/tests/test-driver-custom-multitest-recheck.test
@@ -0,0 +1,223 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: try the "recheck" functionality with test protocols
+# that allow multiple testcases in a single test script. This test not
+# only checks implementation details in Automake's custom test drivers
+# support, but also serves as a "usability test" for our APIs.
+# See also related tests `test-driver-custom-multitest-recheck2.test'
+# and `parallel-tests-recheck-override.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = a.test b.test c.test d.test
+END
+
+cat > a.test << 'END'
+#! /bin/sh
+echo PASS: aa
+echo PASS: AA
+: > a.run
+END
+
+cat > b.test << 'END'
+#! /bin/sh
+echo PASS:
+if test -f b.ok; then
+ echo PASS:
+else
+ echo FAIL:
+fi
+: > b.run
+END
+
+cat > c.test << 'END'
+#! /bin/sh
+if test -f c.pass; then
+ echo PASS: c0
+else
+ echo FAIL: c0
+fi
+if test -f c.xfail; then
+ echo XFAIL: c1
+else
+ echo XPASS: c1
+fi
+echo XFAIL: c2
+: > c.run
+END
+
+cat > d.test << 'END'
+#! /bin/sh
+echo SKIP: who cares ...
+(. ./d.extra) || echo FAIL: d.extra failed
+: > d.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+do_recheck ()
+{
+ case $* in
+ --fail) on_bad_rc='&&';;
+ --pass) on_bad_rc='||';;
+ *) fatal_ "invalid usage of function 'do_recheck'";;
+ esac
+ rm -f *.run
+ eval "\$MAKE recheck >stdout $on_bad_rc { cat stdout; ls -l; Exit 1; }; :"
+ cat stdout; ls -l
+}
+
+do_count ()
+{
+ pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR
+ eval "$@"
+ $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout || : # For debugging.
+ test `grep -c '^PASS:' stdout` -eq $pass
+ test `grep -c '^FAIL:' stdout` -eq $fail
+ test `grep -c '^XPASS:' stdout` -eq $xpass
+ test `grep -c '^XFAIL:' stdout` -eq $xfail
+ test `grep -c '^SKIP:' stdout` -eq $skip
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ : A "make recheck" in a clean tree should run no tests.
+ do_recheck --pass
+ cat test-suite.log
+ test ! -r a.run
+ test ! -r a.log
+ test ! -r b.run
+ test ! -r b.log
+ test ! -r c.run
+ test ! -r c.log
+ test ! -r d.run
+ test ! -r d.log
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+
+ : Run the tests for the first time.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ ls -l
+ # All the test scripts should have run.
+ test -f a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ do_count pass=3 fail=3 xpass=1 xfail=1 skip=1
+
+ : Let us make b.test pass.
+ echo OK > b.ok
+ do_recheck --fail
+ # a.test has been successful the first time, so no need to re-run it.
+ # Similar considerations apply to similar checks, below.
+ test ! -r a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ do_count pass=2 fail=2 xpass=1 xfail=1 skip=1
+
+ : Let us make the first part of c.test pass.
+ echo OK > c.pass
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ do_count pass=1 fail=1 xpass=1 xfail=1 skip=1
+
+ : Let us make also the second part of c.test pass.
+ echo KO > c.xfail
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ do_count pass=1 fail=1 xpass=0 xfail=2 skip=1
+
+ : Nothing changed, so only d.test should be run.
+ for i in 1 2; do
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ do_count pass=0 fail=1 xpass=0 xfail=0 skip=1
+ done
+
+ : Let us make d.test run more testcases, and experience _more_ failures.
+ unindent > d.extra <<'END'
+ echo SKIP: s
+ echo FAIL: f 1
+ echo PASS: p 1
+ echo FAIL: f 2
+ echo XPASS: xp
+ echo FAIL: f 3
+ echo FAIL: f 4
+ echo PASS: p 2
+END
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ do_count pass=2 fail=4 xpass=1 xfail=0 skip=2
+
+ : Let us finally make d.test pass.
+ echo : > d.extra
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=1
+
+ : All tests have been successful or skipped, nothing should be re-run.
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test ! -r d.run
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/test-driver-custom-multitest-recheck2.test b/tests/test-driver-custom-multitest-recheck2.test
new file mode 100755
index 000000000..a18bdd885
--- /dev/null
+++ b/tests/test-driver-custom-multitest-recheck2.test
@@ -0,0 +1,172 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: try the "recheck" functionality with test protocols
+# that allow multiple testcases in a single test script. In particular,
+# check that this still works when we override $(TESTS) and $(TEST_LOGS)
+# at make runtime.
+# See also related tests `test-driver-custom-multitest-recheck.test' and
+# `parallel-tests-recheck-override.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = a.test b.test c.test
+END
+
+cat > a.test << 'END'
+#! /bin/sh
+echo PASS: 1
+echo PASS: 2
+: > a.run
+END
+
+cat > b.test << 'END'
+#! /bin/sh
+echo SKIP: b0
+if test -f b.ok; then
+ echo XFAIL: b1
+else
+ echo FAIL: b2
+fi
+: > b.run
+END
+
+cat > c.test << 'END'
+#! /bin/sh
+echo XPASS: xp
+: > c.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+do_count ()
+{
+ pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR
+ eval "$@"
+ $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout || : # For debugging.
+ test `grep -c '^PASS:' stdout` -eq $pass
+ test `grep -c '^FAIL:' stdout` -eq $fail
+ test `grep -c '^XPASS:' stdout` -eq $xpass
+ test `grep -c '^XFAIL:' stdout` -eq $xfail
+ test `grep -c '^SKIP:' stdout` -eq $skip
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ : Run the tests for the first time.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ # All the test scripts should have run.
+ test -f a.run
+ test -f b.run
+ test -f c.run
+ do_count pass=2 fail=1 xpass=1 xfail=0 skip=1
+
+ rm -f *.run
+
+ : An empty '$(TESTS)' or '$(TEST_LOGS)' means that no test should be run.
+ for var in TESTS TEST_LOGS; do
+ env "$var=" $MAKE -e recheck >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ done
+ unset var
+
+ : a.test was sucessfull the first time, no need to re-run it.
+ env TESTS=a.test $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+
+ : b.test failed, it should be re-run. And make it pass this time.
+ echo OK > b.ok
+ TEST_LOGS=b.log $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ test ! -r a.run
+ test -f b.run
+ test ! -r c.run
+ do_count pass=0 fail=0 xpass=0 xfail=1 skip=1
+
+ rm -f *.run
+
+ : No need to re-run a.test or b.test anymore.
+ TEST_LOGS=b.log $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ TESTS='a.test b.test' $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+
+ # An XPASS should count a failure.
+ env TEST_LOGS='a.log c.log' $MAKE -e recheck >stdout \
+ && { cat stdout; Exit 1; }
+ cat stdout
+ do_count pass=0 fail=0 xpass=1 xfail=0 skip=0
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ rm -f *.run
+ env TESTS='c.test b.test' $MAKE -e recheck >stdout \
+ && { cat stdout; Exit 1; }
+ cat stdout
+ do_count pass=0 fail=0 xpass=1 xfail=0 skip=0
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/test-driver-custom-multitest.test b/tests/test-driver-custom-multitest.test
new file mode 100755
index 000000000..a1c1010b3
--- /dev/null
+++ b/tests/test-driver-custom-multitest.test
@@ -0,0 +1,191 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: check that we can easily support test protocols
+# that allow multiple testcases in a single test script. This test not
+# only checks implementation details in Automake's custom test drivers
+# support, but also serves as a "usability test" for our APIs.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+
+TESTS = \
+ pass.t \
+ fail.t \
+ fail2.t \
+ pass-fail.t \
+ pass4-skip.t \
+ pass3-skip2-xfail.t \
+ pass-xpass-fail-xfail-skip.t
+END
+
+expected_pass=10
+expected_fail=5
+expected_skip=4
+expected_xfail=2
+expected_xpass=1
+
+cat > pass.t << 'END'
+echo %% pass %%
+echo PASS: pass
+END
+
+cat > fail.t << 'END'
+echo %% fail %%
+echo FAIL: fail
+END
+
+cat > fail2.t << 'END'
+echo %% fail2 %%
+echo FAIL: stdout >&1
+echo FAIL: stderr >&2
+echo :PASS: this should be ignored
+END
+
+cat > pass-fail.t << 'END'
+echo %% pass-fail %%
+echo 'FAIL: this fails :-('
+echo 'some randome message'
+echo 'some randome warning' >&2
+echo 'PASS: this passes :-)'
+echo 'INFO: blah'
+echo 'WARNING: blah blah' >&2
+END
+
+cat > pass4-skip.t << 'END'
+echo %% pass4-skip %%
+echo PASS: on stdout >&1
+echo PASS: on stderr >&2
+echo PASS: 3
+echo PASS: 4
+echo SKIP: 1
+echo this FAIL: should be ignored
+echo FAIL as should this
+exit 99
+END
+
+cat > pass3-skip2-xfail.t << 'END'
+echo %% pass4-skip2-xfail %%
+echo 'PASS: -v'
+echo 'PASS: --verbose'
+echo 'SKIP: Oops, unsupported system.'
+echo 'PASS: -#-#-#-'
+cp || echo "SKIP: cp cannot read users' mind" >&2
+mv || echo "XFAIL: mv cannot read users' mind yet"
+exit 127
+END
+
+cat > pass-xpass-fail-xfail-skip.t << 'END'
+echo PASS:
+echo FAIL:
+echo XFAIL:
+echo XPASS:
+echo SKIP:
+echo %% pass-xpass-fail-xfail-skip %%
+END
+
+chmod a+x *.t
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ $MAKE check >stdout && { cat stdout; cat test-suite.log; Exit 1; }
+ cat stdout
+ cat test-suite.log
+ # Couple of sanity checks. These might need to be updated if the
+ # `trivial-test-driver' script is changed.
+ $FGREP INVALID.NAME stdout test-suite.log && Exit 1
+ test -f BAD.LOG && Exit 1
+ # These log files must all have been created by the testsuite.
+ cat pass.log
+ cat fail.log
+ cat fail2.log
+ cat pass-fail.log
+ cat pass4-skip.log
+ cat pass3-skip2-xfail.log
+ cat pass-xpass-fail-xfail-skip.log
+ # For debugging.
+ $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout
+
+ test `grep -c '^PASS:' stdout` -eq $expected_pass
+ test `grep -c '^FAIL:' stdout` -eq $expected_fail
+ test `grep -c '^XPASS:' stdout` -eq $expected_xpass
+ test `grep -c '^XFAIL:' stdout` -eq $expected_xfail
+ test `grep -c '^SKIP:' stdout` -eq $expected_skip
+
+ grep '^PASS: pass-xpass-fail-xfail-skip.t\, testcase 1' stdout
+ grep '^FAIL: pass-xpass-fail-xfail-skip\.t, testcase 2' stdout
+ grep '^XFAIL: pass-xpass-fail-xfail-skip\.t, testcase 3' stdout
+ grep '^XPASS: pass-xpass-fail-xfail-skip\.t, testcase 4' stdout
+ grep '^SKIP: pass-xpass-fail-xfail-skip\.t, testcase 5' stdout
+
+ # Check testsuite summary printed on console.
+ sed -e 's/[()]/ /g' -e 's/^/ /' stdout > t
+ grep ' 6 of 18 ' t
+ grep ' 1 unexpected pass' t
+ grep ' 4 test.* not run' t
+
+ # Check that the content of, and only of, the test logs with at least
+ # one failing test case has been copied into `test-suite.log'. Note
+ # that test logs containing skipped or failed test cases are *not*
+ # copied into `test-suite.log' -- a behaviour that deliberately differs
+ # from the one of the built-in Automake test drivers.
+ grep '%%' test-suite.log # For debugging.
+ grep '%% fail %%' test-suite.log
+ grep '%% fail2 %%' test-suite.log
+ grep '%% pass-fail %%' test-suite.log
+ grep '%% pass-xpass-fail-xfail-skip %%' test-suite.log
+ test `grep -c '%% ' test-suite.log` -eq 4
+
+ TESTS='pass.t pass3-skip2-xfail.t' $MAKE -e check >stdout \
+ || { cat stdout; cat test-suite.log; Exit 1; }
+ cat test-suite.log
+ cat stdout
+ # For debugging.
+ $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout
+ test `grep -c '^PASS:' stdout` -eq 4
+ test `grep -c '^SKIP:' stdout` -eq 2
+ test `grep -c '^XFAIL:' stdout` -eq 1
+ $EGREP '^(FAIL|XPASS)' stdout && Exit 1
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/test-driver-custom-no-extra-driver.test b/tests/test-driver-custom-no-extra-driver.test
index f4c996e27..28c9a1958 100755
--- a/tests/test-driver-custom-no-extra-driver.test
+++ b/tests/test-driver-custom-no-extra-driver.test
@@ -14,7 +14,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-# Check that auxiliary script 'pt-driver' doesn't get needlessly
+# Check that auxiliary script 'test-driver' doesn't get needlessly
# installed or referenced if it's not used, i.e., if the user has
# defined his own `*LOG_DRIVER' variables.
@@ -56,8 +56,8 @@ $ACLOCAL
for opts in '' '--add-missing' '-a -c'; do
$AUTOMAKE $opts
- $FGREP pt-driver Makefile.in sub1/Makefile.in sub2/Makefile.in && Exit 1
- find . | $FGREP pt-driver && Exit 1
+ $FGREP test-driver Makefile.in sub[12]/Makefile.in && Exit 1
+ find . | $FGREP test-driver && Exit 1
done
:
diff --git a/tests/test-driver-custom-no-html.test b/tests/test-driver-custom-no-html.test
new file mode 100755
index 000000000..9aceae613
--- /dev/null
+++ b/tests/test-driver-custom-no-html.test
@@ -0,0 +1,67 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that custom test drivers does not need to produce sensible
+# reStructuredText output in the test logs. This might be legitimate
+# for drivers that are not interested to support the .log -> HTML
+# conversion offered by Automake.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = ./no-rst
+TESTS = foo.test
+END
+
+: > foo.test
+
+cat > no-rst <<'END'
+#! /bin/sh
+# The genereted log file is deliberately syntactically invalid
+# reStructuredText.
+cat > foo.log <<'EoL'
+SKIP: FooBar
+=============
+
+:test-result: SKIP
+
+--------------
+ dummy title
+EoL
+END
+chmod a+x no-rst
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+VERBOSE=yes $MAKE check
+cat foo.log
+cat test-suite.log
+$FGREP 'dummy title' test-suite.log
+
+# Sanity check: trying to produce HTML output should fail.
+$MAKE check-html >output 2>&1 && { cat output; Exit 1; }
+cat output
+$EGREP 'SEVERE|ERROR' output
+
+:
diff --git a/tests/test-driver-custom-xfail-tests.test b/tests/test-driver-custom-xfail-tests.test
index 0d10594df..4f8bb83ea 100755
--- a/tests/test-driver-custom-xfail-tests.test
+++ b/tests/test-driver-custom-xfail-tests.test
@@ -111,12 +111,27 @@ st=0
"$@" || st=$?
rm -f "$log_file"
case $st,$expect_failure in
- 0,no) echo "PASS: $test_name"; exit 0;;
- 1,no) echo "FAIL: $test_name"; exit 1;;
- 0,yes) echo "XPASS: $test_name"; exit 1;;
- 1,yes) echo "XFAIL: $test_name"; exit 0;;
- *) echo "UNEXPECTED OUTCOME: $test_name"; exit 99;;
-esac | tee "$log_file"
+ 0,no)
+ echo "PASS: $test_name" | tee "$log_file"
+ echo ":test-result: PASS" >> "$log_file"
+ ;;
+ 1,no)
+ echo "FAIL: $test_name" | tee "$log_file"
+ echo ":test-result: FAIL" >> "$log_file"
+ ;;
+ 0,yes)
+ echo "XPASS: $test_name" | tee "$log_file"
+ echo ":test-result: XPASS" >> "$log_file"
+ ;;
+ 1,yes)
+ echo "XFAIL: $test_name" | tee "$log_file"
+ echo ":test-result: XFAIL" >> "$log_file"
+ ;;
+ *)
+ echo "INTERNAL ERROR" >&2
+ exit 99
+ ;;
+esac
END
chmod a+x td
diff --git a/tests/trivial-test-driver b/tests/trivial-test-driver
new file mode 100644
index 000000000..f06eb1c87
--- /dev/null
+++ b/tests/trivial-test-driver
@@ -0,0 +1,101 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#
+# Test driver for a very simple test protocol used by the Automake
+# testsuite to check support for custom test drivers allowing for more
+# test results per test script.
+#
+# The exit status of the wrapped script is ignored. Lines in its stdout
+# and stderr beginning with `PASS', `FAIL', `XFAIL', `XPASS' or `SKIP'
+# count as a test case result with the obviously-corresponding outcome.
+# Every other line is ignored for what concerns the testsuite outcome.
+#
+# This script is used at least by the `driver-custom-multitest*.test'
+# tests.
+#
+
+# Help to avoid typo-related bugs.
+set -u
+
+## Option parsing.
+
+test_name=INVALID.NAME
+log_file=BAD.LOG
+while test $# -gt 0; do
+ case $1 in
+ --test-name) test_name=$2; shift;;
+ --log-file) log_file=$2; shift;;
+ # Ignored.
+ --expect-failure) shift;;
+ --color-tests) shift;;
+ --enable-hard-errors) shift;;
+ # Explicitly terminate option list.
+ --) shift; break;;
+ # Shouldn't happen
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+
+## Run the test script, get test cases results, display them on console.
+
+tmp_out=$log_file-out.tmp
+tmp_res=$log_file-res.tmp
+
+"$@" 2>&1 | tee $tmp_out | (
+ i=0 st=0
+ : > $tmp_res
+ while read line; do
+ case $line in
+ PASS:*|FAIL:*|XPASS:*|XFAIL:*|SKIP:*)
+ i=`expr $i + 1`
+ result=`LC_ALL=C expr "$line" : '\([A-Z]*\):.*'`
+ case $result in FAIL|XPASS) st=1;; esac
+ # Output testcase result to console.
+ echo "$result: $test_name, testcase $i"
+ # Register testcase outcome for the log file.
+ echo ":test-result: $line" >> $tmp_res
+ echo >> $tmp_res
+ ;;
+ esac
+ done
+ exit $st
+)
+
+if test $? -eq 0; then
+ global_result=PASS
+else
+ global_result=FAIL
+fi
+
+## Write the log file.
+
+{
+ echo "$global_result: $test_name"
+ echo "$global_result: $test_name" | sed 's/./=/g'
+ echo
+ cat $tmp_res
+ echo
+ echo --------------------
+ echo
+ cat $tmp_out
+} > $log_file
+rm -f $tmp_out $tmp_res
+
+## And we're done.
+
+exit 0
View Lorry Controller Status