summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Makefile.am1
-rw-r--r--NEWS13
-rwxr-xr-xcontrib/tap-driver.pl (renamed from lib/tap-driver.pl)9
-rw-r--r--doc/automake.texi22
-rwxr-xr-xgen-testsuite-part14
-rw-r--r--lib/Makefile.inc3
-rwxr-xr-xlib/test-driver13
-rw-r--r--t/ax/am-test-lib.sh53
-rw-r--r--t/ax/test-lib.sh3
-rw-r--r--t/list-of-tests.mk1
-rw-r--r--t/parallel-tests-exit-status-reported.sh68
-rw-r--r--t/tap-bad-prog.tap30
-rw-r--r--t/tap-bailout-leading-space.sh9
-rw-r--r--t/tap-signal.tap9
-rw-r--r--t/tap-test-number-0.sh24
-rw-r--r--t/test-driver-cond.sh8
16 files changed, 132 insertions, 148 deletions
diff --git a/Makefile.am b/Makefile.am
index 143308a11..f8b6ac5c6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -92,6 +92,7 @@ ChangeLog:
# Third-party, obsolescent or experimental stuff.
EXTRA_DIST += \
+ contrib/tap-driver.pl \
contrib/check-html.am \
contrib/multilib/README \
contrib/multilib/config-ml.in \
diff --git a/NEWS b/NEWS
index 5cc001977..a6e197946 100644
--- a/NEWS
+++ b/NEWS
@@ -73,6 +73,19 @@ New in 1.15:
(3) the "set -f" and "set +f" shell commands work, and, respectively,
disable and enable shell globbing.
+* Automake-generated testsuites:
+
+ - The default test-driver used by the Automake-generates testsuites now
+ append the result and exit status of each "plain" test to the associated
+ log file (automake bug#118149).
+
+ - The perl implementation of the TAP testsuite driver is no longer
+ installed in the Automake's scripts directory, and is instead just
+ distributed as a "contrib" addition. There should be no reason to
+ use this implementation anyway in real packages, since the awk+shell
+ implementation of the TAP driver (that is documented in the manual)
+ is more portable and has feature parity with the perl implementation.
+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
New in 1.14.1:
diff --git a/lib/tap-driver.pl b/contrib/tap-driver.pl
index aca65fe44..e7e581de4 100755
--- a/lib/tap-driver.pl
+++ b/contrib/tap-driver.pl
@@ -32,7 +32,7 @@ use strict;
use Getopt::Long ();
use TAP::Parser;
-my $VERSION = '2012-02-01.19'; # UTC
+my $VERSION = '2013-12-24.15'; # UTC
my $ME = "tap-driver.pl";
@@ -43,13 +43,13 @@ Usage:
[--enable-hard-errors={yes|no}] [--ignore-exit]
[--diagnostic-string=STRING] [--merge|--no-merge]
[--comments|--no-comments] [--] TEST-COMMAND
-The `--test-name', `--log-file' and `--trs-file' options are mandatory.
+The '--test-name', '--log-file' and '--trs-file' options are mandatory.
END
my $HELP = "$ME: TAP-aware test driver for Automake testsuite harness." .
"\n" . $USAGE;
-# Keep this in sync with `lib/am/check.am:$(am__tty_colors)'.
+# Keep this in sync with 'lib/am/check.am:$(am__tty_colors)'.
my %COLOR = (
red => "\e[0;31m",
grn => "\e[0;32m",
@@ -208,7 +208,6 @@ TEST_RESULTS :
return grep { not $_ eq "PASS" } (keys %test_results_seen);
}
- # FIXME: this can certainly be improved ...
sub get_global_test_result ()
{
return "ERROR"
@@ -465,7 +464,7 @@ sub extract_tap_comment ($)
my $line = shift;
if (index ($line, $diag_string) == 0)
{
- # Strip leading `$diag_string' from `$line'.
+ # Strip leading '$diag_string' from '$line'.
$line = substr ($line, length ($diag_string));
# And strip any leading and trailing whitespace left.
$line =~ s/(?:^\s*|\s*$)//g;
diff --git a/doc/automake.texi b/doc/automake.texi
index 645e7298f..91b4a0a7d 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -1900,9 +1900,6 @@ It is customary to make the first line of @file{Makefile.am} read:
## Process this file with automake to produce Makefile.in
@end example
-@c FIXME discuss putting a copyright into Makefile.am here? I would but
-@c I don't know quite what to say.
-
@c FIXME document customary ordering of Makefile.am here!
@@ -8821,9 +8818,6 @@ error} happens when e.g., the set-up of a test case scenario fails, or when
some other unexpected or highly undesirable condition is encountered (for
example, the program under test experiences a segmentation fault).
-@emph{TODO}: Links to other test harnesses (esp. those sharing our
-terminology)?
-
@node Simple Tests
@section Simple Tests
@@ -8956,8 +8950,6 @@ flag on file descriptors opened with the @command{exec} builtin, thus
rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;}
ineffectual. This issue also affects some Bourne shells, such as the
HP-UX's @command{/bin/sh},
-@c FIXME: should we offer a link to the relevant discussions on the
-@c bug-autoconf list?
@c Keep in sync with tests-environment-backcompat.sh
@example
@@ -9017,7 +9009,6 @@ The serial test harness is enabled by the Automake option
@option{serial-tests}. It operates by simply running the tests serially,
one at the time, without any I/O redirection. It's up to the user to
implement logging of tests' output, if that's requited or desired.
-@c TODO: give an example of how this can be done.
For historical and implementation reasons, the @code{AM_TESTS_ENVIRONMENT}
variable is @emph{not} supported by this harness (it will be silently
@@ -9086,8 +9077,6 @@ to a per-test log file, so that parallel execution does not produce
intermingled output. The output from failed tests is collected in the
@file{test-suite.log} file. If the variable @samp{VERBOSE} is set, this
file is output after the summary.
-@c FIXME: we should be clearer about what we mean exactly here ...
-For best results, the tests should be verbose by default now.
@vindex TEST_EXTENSIONS
@vindex TEST_LOGS
@@ -9596,12 +9585,6 @@ other end, if you are using a known and widespread test protocol with
well-established implementations, being consistent with those
implementations' output might be a good idea too.
-@c TODO: Give an example, maybe inspired to py.test-style output.
-@c TODO: That is a good idea because it shows a test driver that allows
-@c TODO: for different levels of verbosity in the progress output (could
-@c TODO: be implemented either using a driver cmdline flag, or an
-@c TODO: environment variable, or both).
-
@node Using the TAP test protocol
@section Using the TAP test protocol
@@ -9740,7 +9723,7 @@ echo ok 1
# Exit with error, even if all the tests have been successful.
exit 7
-% @kbd{cp @var{PREFIX}/share/automake-@var{APIVERSION}/tap-driver.pl .}
+% @kbd{cp @var{PREFIX}/share/automake-@var{APIVERSION}/tap-driver.sh .}
% @kbd{autoreconf -vi && ./configure && make check}
...
PASS: foo.test 1 - Swallows fly
@@ -10836,9 +10819,6 @@ they all have their serious drawbacks and limitations. That's why
automake provides support for a more advanced and flexible way of
obtaining quieter output from @command{make} (for most rules at least).
-@c TODO: Maybe describe in brief the precedent set by the build system
-@c of the Linux Kernel, from which Automake took inspiration ... Links?
-
To give the gist of what Automake can do in this respect, here is a simple
comparison between a typical @command{make} output (where silent rules
are disabled) and one with silent rules enabled:
diff --git a/gen-testsuite-part b/gen-testsuite-part
index 3bd5c9f01..5bd1b4e96 100755
--- a/gen-testsuite-part
+++ b/gen-testsuite-part
@@ -241,20 +241,6 @@ my %test_generators =
shell_setup_code =>
'am_test_prefer_config_shell=yes',
},
- #
- # Tests on tap support should be run with both the perl and awk
- # implementations of the TAP driver (they run with the awk one
- # by default).
- #
- perl_tap_driver =>
- {
- line_matcher =>
- qr<(?:\bfetch_tap_driver\b|[\s/]tap-setup\.sh\b)>,
- line_rejecter =>
- qr/\bam_tap_implementation=/,
- shell_setup_code =>
- 'am_tap_implementation=perl',
- },
);
#--------------------------------------------------------------------------
diff --git a/lib/Makefile.inc b/lib/Makefile.inc
index d1971f55f..8eed3038e 100644
--- a/lib/Makefile.inc
+++ b/lib/Makefile.inc
@@ -40,8 +40,7 @@ dist_script_DATA = \
%D%/py-compile \
%D%/ar-lib \
%D%/test-driver \
- %D%/tap-driver.sh \
- %D%/tap-driver.pl
+ %D%/tap-driver.sh
install-data-hook:
@$(POST_INSTALL)
diff --git a/lib/test-driver b/lib/test-driver
index d30605660..110eec409 100755
--- a/lib/test-driver
+++ b/lib/test-driver
@@ -106,11 +106,14 @@ trap "st=143; $do_exit" 15
# Test script is run here.
"$@" >$log_file 2>&1
estatus=$?
+
if test $enable_hard_errors = no && test $estatus -eq 99; then
- estatus=1
+ tweaked_estatus=1
+else
+ tweaked_estatus=$estatus
fi
-case $estatus:$expect_failure in
+case $tweaked_estatus:$expect_failure in
0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
0:*) col=$grn res=PASS recheck=no gcopy=no;;
77:*) col=$blu res=SKIP recheck=no gcopy=yes;;
@@ -119,6 +122,12 @@ case $estatus:$expect_failure in
*:*) col=$red res=FAIL recheck=yes gcopy=yes;;
esac
+# Report the test outcome and exit status in the logs, so that one can
+# know whether the test passed or failed simply by looking at the '.log'
+# file, without the need of also peaking into the corresponding '.trs'
+# file (automake bug#11814).
+echo "$res $test_name (exit status: $estatus)" >>$log_file
+
# Report outcome to console.
echo "${col}${res}${std}: $test_name"
diff --git a/t/ax/am-test-lib.sh b/t/ax/am-test-lib.sh
index 26e58ef04..cee5c8da8 100644
--- a/t/ax/am-test-lib.sh
+++ b/t/ax/am-test-lib.sh
@@ -583,17 +583,31 @@ count_test_results ()
# of /bin/sh.
get_shell_script ()
{
- test ! -f "$1" || rm -f "$1" || return 99
+ am_source=$1 am_target=${2-$1}
+ test ! -f "$am_target" || rm -f "$am_target" || return 99
if test x"$am_test_prefer_config_shell" = x"yes"; then
- sed "1s|#!.*|#! $SHELL|" "$am_scriptdir/$1" > "$1" \
- && chmod a+x "$1" \
+ sed "1s|#!.*|#! $SHELL|" "$am_scriptdir/$am_source" > "$am_target" \
+ && chmod a+x "$am_target" \
|| return 99
else
- cp -f "$am_scriptdir/$1" . || return 99
+ cp -f "$am_scriptdir/$am_source" "$am_target" || return 99
fi
- sed 10q "$1" # For debugging.
+ sed 10q "$am_target" # For debugging.
+ unset am_target am_source
}
+# fetch_tap_driver
+# ----------------
+# Fetch the Automake-provided TAP driver from the 'lib/' directory into
+# the current directory, and edit its shebang line so that it will be
+# run with the proper shell.
+fetch_tap_driver ()
+{
+ AM_TAP_AWK=$AWK; export AM_TAP_AWK
+ get_shell_script tap-driver.sh tap-driver
+}
+
+
# require_xsi SHELL
# -----------------
# Skip the test if the given shell fails to support common XSI constructs.
@@ -612,35 +626,6 @@ xsi_shell_code='
&& eval '\''test $(( 1 + 1 )) -eq 2 \
&& test "${#_lt_dummy}" -eq 5'\'
-# fetch_tap_driver
-# ----------------
-# Fetch the Automake-provided TAP driver from the 'lib/' directory into
-# the current directory, and edit its shebang line so that it will be
-# run with the perl interpreter determined at configure time.
-fetch_tap_driver ()
-{
- # TODO: we should devise a way to make the shell TAP driver tested also
- # TODO: with /bin/sh, for better coverage.
- case $am_tap_implementation in
- # Extra quoting required to avoid maintainer-check spurious failures.
- 'perl')
- $PERL -MTAP::Parser -e 1 \
- || skip_all_ "cannot import TAP::Parser perl module"
- sed "1s|#!.*|#! $PERL -w|" "$am_scriptdir"/tap-driver.pl >tap-driver
- ;;
- shell)
- AM_TAP_AWK=$AWK; export AM_TAP_AWK
- sed "1s|#!.*|#! $SHELL|" "$am_scriptdir"/tap-driver.sh >tap-driver
- ;;
- *)
- fatal_ "invalid \$am_tap_implementation '$am_tap_implementation'" ;;
- esac \
- && chmod a+x tap-driver \
- || framework_failure_ "couldn't fetch $am_tap_implementation TAP driver"
- sed 10q tap-driver # For debugging.
-}
-am_tap_implementation=${am_tap_implementation-shell}
-
# $PYTHON and support for PEP-3147. Needed to check our python-related
# install rules.
python_has_pep3147 ()
diff --git a/t/ax/test-lib.sh b/t/ax/test-lib.sh
index aa015d68d..b8c72533c 100644
--- a/t/ax/test-lib.sh
+++ b/t/ax/test-lib.sh
@@ -254,7 +254,7 @@ am_exit_trap ()
# behaviour, while from time to time useful to developers, is not
# meant to be enabled by default, as it could cause spurious failures
# in the wild. Thus it will be enabled only when the variable
- # "am_explicit_skips" is set to a "true" value.
+ # 'am_explicit_skips' is set to a "true" value.
case $am_explicit_skips in
[yY]|[yY]es|1)
if test $exit_status -eq 77 && test $am__test_skipped != yes; then
@@ -266,7 +266,6 @@ am_exit_trap ()
fi
am_keeping_testdirs || rm_rf_ $am_test_subdir
set +x
- echo "$me: exit $exit_status"
# Spurious escaping to ensure we do not call our "exit" alias.
\exit $exit_status
}
diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk
index 462497ee3..ba6578903 100644
--- a/t/list-of-tests.mk
+++ b/t/list-of-tests.mk
@@ -765,6 +765,7 @@ t/parallel-tests-basics.sh \
t/parallel-tests-concurrency.sh \
t/parallel-tests-concurrency-2.sh \
t/parallel-tests-empty.sh \
+t/parallel-tests-exit-status-reported.sh \
t/parallel-tests-generated-and-distributed.sh \
t/parallel-tests-recheck.sh \
t/parallel-tests-trailing-whitespace.sh \
diff --git a/t/parallel-tests-exit-status-reported.sh b/t/parallel-tests-exit-status-reported.sh
new file mode 100644
index 000000000..d40f1f58a
--- /dev/null
+++ b/t/parallel-tests-exit-status-reported.sh
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2013 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# The exit status of a test should be reported in the test logs, so
+# that one can see at a glance whether the test has succeeded or failed,
+# without having to look also into the corresponding .trs file.
+# See automake bug#11814.
+
+. test-init.sh
+
+echo AC_OUTPUT >> configure.ac
+
+echo XFAIL_TESTS = t3.test t00.test > Makefile.am
+echo TESTS = t00.test >> Makefile.am
+for s in 0 1 2 3 5 77 78 99 100 126 127; do
+ echo "TESTS += t${s}.test" >> Makefile.am
+ cat > t${s}.test <<END
+#!/bin/sh
+printf "%s\\n%s\\n" 'random' 'will exit with status $s'
+exit $s
+END
+done
+cp t0.test t00.test
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+run_make -e FAIL check
+ls -l # For debugging.
+
+match_result ()
+{
+ cat "$1.log" # For debugging.
+ test $(wc -l <"$1.log") -eq 3
+ sed -n '$p' "$1.log" | grep "^$2 $1\\.test (exit status: $3)$"
+}
+
+match_result t0 PASS 0
+match_result t00 XPASS 0
+match_result t1 FAIL 1
+match_result t2 FAIL 2
+match_result t3 XFAIL 3
+match_result t5 FAIL 5
+match_result t77 SKIP 77
+match_result t78 FAIL 78
+match_result t99 ERROR 99
+match_result t100 FAIL 100
+match_result t126 FAIL 126
+match_result t127 FAIL 127
+
+:
diff --git a/t/tap-bad-prog.tap b/t/tap-bad-prog.tap
index 490cd4a78..9225c38ed 100644
--- a/t/tap-bad-prog.tap
+++ b/t/tap-bad-prog.tap
@@ -73,35 +73,13 @@ else
fi
# Check that no spurious test result is reported. This is lower-priority
-# (and in fact the check currently fails for our awk-based driver).
-directive=
-if test $am_tap_implementation = shell; then
- directive=TODO
-else
- # Older versions of IPC::Open3 (e.g., version 1.05 on perl 5.12.4 or
- # version 1.0103 on perl 5.6.2) fail to properly trap errors in exec(2)
- # calls in the child process; hence, the TAP driver cannot be properly
- # informed of such error.
- if $PERL -w -e '
- use IPC::Open3 qw/open3/;
- $@ = "";
- eval { open3(*STDIN, *STDOUT, *STDERR, "am--no-such-command") };
- $@ =~ m/\bopen3:.*am--no-such-command/
- or die "Bad \$@ value: \"$@\"\n";
- '; then
- : # OK. IPC::Open3 should be good enough.
- else
- for s in '"missing plan" message' 'results'; do
- skip_ -r "IPC::Open3 not good enough" "no spurious $s"
- done
- exit 0
- fi
-fi
+# (and in fact the check currently fails).
command_ok_ 'no spurious "missing plan" message' \
- -D "$directive" -- not grep 'missing.* plan' stdout
+ -D TODO -- not grep 'missing.* plan' stdout
+
command_ok_ 'no spurious results' \
- -D "$directive" -r 'still get "missing plan"' \
+ -D TODO -r 'still get "missing plan"' \
count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
:
diff --git a/t/tap-bailout-leading-space.sh b/t/tap-bailout-leading-space.sh
index 120ae0340..7a55a1fba 100644
--- a/t/tap-bailout-leading-space.sh
+++ b/t/tap-bailout-leading-space.sh
@@ -14,15 +14,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-# Older versions of prove and TAP::Harness (e.g., 3.17) didn't recognize
-# a "Bail out!" directive that was preceded by whitespace, but more modern
-# versions (e.g., 3.23) do. So we leave this behaviour undefined for the
-# perl implementation of the Automake TAP driver, but expect the latter,
-# "more modern" behaviour in our awk TAP driver.
+# A "Bail out!" directive that is preceded by whitespace should still
+# be recognized.
-am_tap_implementation=shell
. test-init.sh
-
. tap-setup.sh
cat > a.test <<END
diff --git a/t/tap-signal.tap b/t/tap-signal.tap
index f1a81308b..70fe5c19d 100644
--- a/t/tap-signal.tap
+++ b/t/tap-signal.tap
@@ -93,13 +93,8 @@ signal_caught ()
esac
wbound_re="($|[^a-zA-Z0-9_-])"
pfx_re="^ERROR: signal-$numeric\\.test"
- case $am_tap_implementation in
- # Dummy escape to please maintainer-check.
- per\l) rx="$pfx_re - terminated by signal $sig_re$";;
- shell) rx="$pfx_re .*terminated by signal $sig_re$wbound_re";;
- *) fatal_ "invalid \$am_tap_implementation '$am_tap_implementation'";;
- esac
- desc="TAP driver catch test termination by signal SIG$symbolic"
+ rx="${pfx_re} .*terminated by signal ${sig_re}${wbound_re}"
+ desc="TAP driver catch test termination by signal SIG${symbolic}"
case " $blocked_signals " in
*" $numeric "*) skip_ -r "SIG$symbolic is blocked" "$desc" ;;
*) command_ok_ "$desc" env LC_ALL=C $EGREP "$rx" stdout ;;
diff --git a/t/tap-test-number-0.sh b/t/tap-test-number-0.sh
index b5b161b30..d447ebe07 100644
--- a/t/tap-test-number-0.sh
+++ b/t/tap-test-number-0.sh
@@ -19,30 +19,6 @@
# This is consistent with the behaviour of the 'prove' utility.
. test-init.sh
-
-if test $am_tap_implementation = perl; then
- $PERL -MTAP::Parser -e 1 \
- || skip_ "cannot import TAP::Parser perl module"
- if $PERL -w -e '
- use warnings FATAL => "all"; use strict;
- use TAP::Parser;
- my $parser = TAP::Parser->new({tap => "1..1\n" . "ok 0\n"});
- my $result = $parser->next;
- $result->is_plan or die "first line is not TAP plan";
- $result = $parser->next;
- $result->is_test or die "second line is not TAP test result";
- my $testno = $result->number;
- $parser->next and die "unexpected further TAP stream";
- exit ($testno == 0 ? 0 : 77);
- '; then
- : # Nothing to do.
- elif test $? -eq 77; then
- skip_ 'TAP::Parser bug: test number 0 gets relabelled as 1'
- else
- fatal_ "error analyzing TAP::Parser module for bugs"
- fi
-fi
-
. tap-setup.sh
cat > a.test <<END
diff --git a/t/test-driver-cond.sh b/t/test-driver-cond.sh
index febb9d2f8..9b50b1c0a 100644
--- a/t/test-driver-cond.sh
+++ b/t/test-driver-cond.sh
@@ -22,8 +22,8 @@
cp "$am_testaux_srcdir"/trivial-test-driver . \
|| fatal_ "failed to fetch auxiliary script trivial-test-driver"
-cp "$am_scriptdir"/tap-driver.pl . \
- || fatal_ "failed to fetch auxiliary script tap-driver.pl"
+cp "$am_scriptdir"/tap-driver.sh . \
+ || fatal_ "failed to fetch auxiliary script tap-driver.sh"
cat >> configure.ac << END
AM_CONDITIONAL([COND1], [:])
@@ -40,7 +40,7 @@ $AUTOCONF
cat > Makefile.am << 'END'
TESTS = foo bar.test baz.sh
-EXTRA_DIST = $(TESTS) tap-driver.pl trivial-test-driver
+EXTRA_DIST = $(TESTS) tap-driver.sh trivial-test-driver
TEST_EXTENSIONS = .test .sh
LOG_DRIVER =
SH_LOG_DRIVER = $(tap_rulez)
@@ -49,7 +49,7 @@ LOG_DRIVER += @my_LOG_DRIVER@
if COND2
tap_rulez = false
else !COND2
-tap_rulez = $(PERL) $(srcdir)/tap-driver.pl
+tap_rulez = $(PERL) $(srcdir)/tap-driver.sh
endif !COND2
endif COND1
END
View Lorry Controller Status