From a6d6734fca39448770365e77acb562e6c42a7ea2 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Tue, 24 Dec 2013 16:27:28 +0100 Subject: TAP driver: remove perl implementation (move it into contrib/) That implementation was only meant as a standard against which the portable awk+shell implementation was to be measured. Now, since Automake 1.12, the latter implementation is fully functional and already used in the wild, and in fact feature-par with the perl implementation. So the perl implementation is now just slowing down and complicating our testsuite. Let's move it to 'contrib/' (we don't want to remove it, in case someone is actually using it in the wild). * lib/tap-driver.pl: Move it ... * contrib/tap-driver.pl: ... here. While at it, convert quoting `like this' to quoting 'like this', and remove an obsolescent FIXME comment. * lib/Makefile.inc (dist_script_DATA): Drop '%D%/tap-driver.pl'. * Makefile.am (EXTRA_DIST): Add 'contrib/tap-driver.pl'. * doc/automake.texi: Remove one stray reference to 'tap-driver.pl', and reference 'tap-driver.sh' instead, as intended. * t/ax/am-test-lib.sh ($am_tap_implementation): Delete definition and uses. (fetch_tap_driver): Simplify to unconditionally assume the shell+awk implementation of the TAP driver is used. (get_shell_script): Make more flexible so that it can cater to the needs of 'fetch_tap_driver()'. * t/tap-bad-prog.tap: Likewise. * t/tap-bailout-leading-space.sh: Likewise. * t/tap-signal.tap: Likewise. * t/tap-test-number-0.sh: Likewise. * t/test-driver-cond.sh: Use 'tap-driver.sh' instead of 'tap-driver.pl'. * gen-testsuite-part (%test_generators): Do not generate sister tests that use the perl TAP driver rather than the shell+awk one. * NEWS: Update. Signed-off-by: Stefano Lattarini --- Makefile.am | 1 + NEWS | 9 + contrib/tap-driver.pl | 563 ++++++++++++++++++++++++++++++++++++++++ doc/automake.texi | 2 +- gen-testsuite-part | 14 - lib/Makefile.inc | 3 +- lib/tap-driver.pl | 564 ----------------------------------------- t/ax/am-test-lib.sh | 53 ++-- t/tap-bad-prog.tap | 30 +-- t/tap-bailout-leading-space.sh | 9 +- t/tap-signal.tap | 9 +- t/tap-test-number-0.sh | 24 -- t/test-driver-cond.sh | 8 +- 13 files changed, 606 insertions(+), 683 deletions(-) create mode 100755 contrib/tap-driver.pl delete mode 100755 lib/tap-driver.pl diff --git a/Makefile.am b/Makefile.am index 143308a11..f8b6ac5c6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -92,6 +92,7 @@ ChangeLog: # Third-party, obsolescent or experimental stuff. EXTRA_DIST += \ + contrib/tap-driver.pl \ contrib/check-html.am \ contrib/multilib/README \ contrib/multilib/config-ml.in \ diff --git a/NEWS b/NEWS index 5cc001977..24dafddb2 100644 --- a/NEWS +++ b/NEWS @@ -73,6 +73,15 @@ New in 1.15: (3) the "set -f" and "set +f" shell commands work, and, respectively, disable and enable shell globbing. +* Automake-generated testsuite: + + - The perl implementation of the TAP testsuite driver is no longer + installed in the Automake's scripts directory, and is instead just + distributed as a "contrib" addition. There should be no reason to + use this implementation anyway in real packages, since the awk+shell + implementation of the TAP driver (that is documented in the manual) + is more portable and has feature parity with the perl implementation. + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New in 1.14.1: diff --git a/contrib/tap-driver.pl b/contrib/tap-driver.pl new file mode 100755 index 000000000..e7e581de4 --- /dev/null +++ b/contrib/tap-driver.pl @@ -0,0 +1,563 @@ +#! /usr/bin/env perl +# Copyright (C) 2011-2013 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to or send patches to +# . + +# ---------------------------------- # +# Imports, static data, and setup. # +# ---------------------------------- # + +use warnings FATAL => 'all'; +use strict; +use Getopt::Long (); +use TAP::Parser; + +my $VERSION = '2013-12-24.15'; # UTC + +my $ME = "tap-driver.pl"; + +my $USAGE = <<'END'; +Usage: + tap-driver --test-name=NAME --log-file=PATH --trs-file=PATH + [--expect-failure={yes|no}] [--color-tests={yes|no}] + [--enable-hard-errors={yes|no}] [--ignore-exit] + [--diagnostic-string=STRING] [--merge|--no-merge] + [--comments|--no-comments] [--] TEST-COMMAND +The '--test-name', '--log-file' and '--trs-file' options are mandatory. +END + +my $HELP = "$ME: TAP-aware test driver for Automake testsuite harness." . + "\n" . $USAGE; + +# Keep this in sync with 'lib/am/check.am:$(am__tty_colors)'. +my %COLOR = ( + red => "\e[0;31m", + grn => "\e[0;32m", + lgn => "\e[1;32m", + blu => "\e[1;34m", + mgn => "\e[0;35m", + brg => "\e[1m", + std => "\e[m", +); + +# It's important that NO_PLAN evaluates "false" as a boolean. +use constant NO_PLAN => 0; +use constant EARLY_PLAN => 1; +use constant LATE_PLAN => 2; + +# ------------------- # +# Global variables. # +# ------------------- # + +my $testno = 0; # Number of test results seen so far. +my $bailed_out = 0; # Whether a "Bail out!" directive has been seen. +my $parser; # TAP parser object (will be initialized later). + +# Whether the TAP plan has been seen or not, and if yes, which kind +# it is ("early" is seen before any test result, "late" otherwise). +my $plan_seen = NO_PLAN; + +# ----------------- # +# Option parsing. # +# ----------------- # + +my %cfg = ( + "color-tests" => 0, + "expect-failure" => 0, + "merge" => 0, + "comments" => 0, + "ignore-exit" => 0, +); + +my $test_script_name = undef; +my $log_file = undef; +my $trs_file = undef; +my $diag_string = "#"; + +Getopt::Long::GetOptions + ( + 'help' => sub { print $HELP; exit 0; }, + 'version' => sub { print "$ME $VERSION\n"; exit 0; }, + 'test-name=s' => \$test_script_name, + 'log-file=s' => \$log_file, + 'trs-file=s' => \$trs_file, + 'color-tests=s' => \&bool_opt, + 'expect-failure=s' => \&bool_opt, + 'enable-hard-errors=s' => sub {}, # No-op. + 'diagnostic-string=s' => \$diag_string, + 'comments' => sub { $cfg{"comments"} = 1; }, + 'no-comments' => sub { $cfg{"comments"} = 0; }, + 'merge' => sub { $cfg{"merge"} = 1; }, + 'no-merge' => sub { $cfg{"merge"} = 0; }, + 'ignore-exit' => sub { $cfg{"ignore-exit"} = 1; }, + ) or exit 1; + +# ------------- # +# Prototypes. # +# ------------- # + +sub add_test_result ($); +sub bool_opt ($$); +sub colored ($$); +sub copy_in_global_log (); +sub decorate_result ($); +sub extract_tap_comment ($); +sub finish (); +sub get_global_test_result (); +sub get_test_exit_message (); +sub get_test_results (); +sub handle_tap_bailout ($); +sub handle_tap_plan ($); +sub handle_tap_result ($); +sub is_null_string ($); +sub main (@); +sub must_recheck (); +sub report ($;$); +sub setup_io (); +sub setup_parser (@); +sub stringify_result_obj ($); +sub testsuite_error ($); +sub trap_perl_warnings_and_errors (); +sub write_test_results (); +sub yn ($); + +# -------------- # +# Subroutines. # +# -------------- # + +sub bool_opt ($$) +{ + my ($opt, $val) = @_; + if ($val =~ /^(?:y|yes)\z/i) + { + $cfg{$opt} = 1; + } + elsif ($val =~ /^(?:n|no)\z/i) + { + $cfg{$opt} = 0; + } + else + { + die "$ME: invalid argument '$val' for option '$opt'\n"; + } +} + +# If the given string is undefined or empty, return true, otherwise +# return false. This function is useful to avoid pitfalls like: +# if ($message) { print "$message\n"; } +# which wouldn't print anything if $message is the literal "0". +sub is_null_string ($) +{ + my $str = shift; + return ! (defined $str and length $str); +} + +# Convert a boolean to a "yes"/"no" string. +sub yn ($) +{ + my $bool = shift; + return $bool ? "yes" : "no"; +} + +TEST_RESULTS : +{ + my (@test_results_list, %test_results_seen); + + sub add_test_result ($) + { + my $res = shift; + push @test_results_list, $res; + $test_results_seen{$res} = 1; + } + + sub get_test_results () + { + return @test_results_list; + } + + # Whether the test script should be re-run by "make recheck". + sub must_recheck () + { + return grep { !/^(?:XFAIL|PASS|SKIP)$/ } (keys %test_results_seen); + } + + # Whether the content of the log file associated to this test should + # be copied into the "global" test-suite.log. + sub copy_in_global_log () + { + return grep { not $_ eq "PASS" } (keys %test_results_seen); + } + + sub get_global_test_result () + { + return "ERROR" + if $test_results_seen{"ERROR"}; + return "FAIL" + if $test_results_seen{"FAIL"} || $test_results_seen{"XPASS"}; + return "SKIP" + if scalar keys %test_results_seen == 1 && $test_results_seen{"SKIP"}; + return "PASS"; + } + +} + +sub write_test_results () +{ + open RES, ">", $trs_file or die "$ME: opening $trs_file: $!\n"; + print RES ":global-test-result: " . get_global_test_result . "\n"; + print RES ":recheck: " . yn (must_recheck) . "\n"; + print RES ":copy-in-global-log: " . yn (copy_in_global_log) . "\n"; + foreach my $result (get_test_results) + { + print RES ":test-result: $result\n"; + } + close RES or die "$ME: closing $trs_file: $!\n"; +} + +sub trap_perl_warnings_and_errors () +{ + $SIG{__WARN__} = $SIG{__DIE__} = sub + { + # Be sure to send the warning/error message to the original stderr + # (presumably the console), not into the log file. + open STDERR, ">&OLDERR"; + die @_; + } +} + +sub setup_io () +{ + # Redirect stderr and stdout to a temporary log file. Save the + # original stdout stream, since we need it to print testsuite + # progress output. Save original stderr stream, so that we can + # redirect warning and error messages from perl there. + open LOG, ">", $log_file or die "$ME: opening $log_file: $!\n"; + open OLDOUT, ">&STDOUT" or die "$ME: duplicating stdout: $!\n"; + open OLDERR, ">&STDERR" or die "$ME: duplicating stdout: $!\n"; + *OLDERR = *OLDERR; # To pacify a "used only once" warning. + trap_perl_warnings_and_errors; + open STDOUT, ">&LOG" or die "$ME: redirecting stdout: $!\n"; + open STDERR, ">&LOG" or die "$ME: redirecting stderr: $!\n"; +} + +sub setup_parser (@) +{ + local $@ = ''; + eval { $parser = TAP::Parser->new ({exec => \@_, merge => $cfg{merge}}) }; + if ($@ ne '') + { + # Don't use the error message in $@ as set by TAP::Parser, since + # currently it's both too generic (at the point of being basically + # useless) and quite long. + report "ERROR", "- couldn't execute test script"; + finish; + } +} + +sub get_test_exit_message () +{ + my $wstatus = $parser->wait; + # Watch out for possible internal errors. + die "$ME: couldn't get the exit status of the TAP producer" + unless defined $wstatus; + # Return an undefined value if the producer exited with success. + return unless $wstatus; + # Otherwise, determine whether it exited with error or was terminated + # by a signal. + use POSIX qw (WIFEXITED WEXITSTATUS WIFSIGNALED WTERMSIG); + if (WIFEXITED ($wstatus)) + { + return sprintf "exited with status %d", WEXITSTATUS ($wstatus); + } + elsif (WIFSIGNALED ($wstatus)) + { + return sprintf "terminated by signal %d", WTERMSIG ($wstatus); + } + else + { + return "terminated abnormally"; + } +} + +sub stringify_result_obj ($) +{ + my $result_obj = shift; + my $COOKED_PASS = $cfg{"expect-failure"} ? "XPASS": "PASS"; + my $COOKED_FAIL = $cfg{"expect-failure"} ? "XFAIL": "FAIL"; + if ($result_obj->is_unplanned || $result_obj->number != $testno) + { + return "ERROR"; + } + elsif ($plan_seen == LATE_PLAN) + { + return "ERROR"; + } + elsif (!$result_obj->directive) + { + return $result_obj->is_ok ? $COOKED_PASS: $COOKED_FAIL; + } + elsif ($result_obj->has_todo) + { + return $result_obj->is_actual_ok ? "XPASS" : "XFAIL"; + } + elsif ($result_obj->has_skip) + { + return $result_obj->is_ok ? "SKIP" : $COOKED_FAIL; + } + die "$ME: INTERNAL ERROR"; # NOTREACHED +} + +sub colored ($$) +{ + my ($color_name, $text) = @_; + return $COLOR{$color_name} . $text . $COLOR{'std'}; +} + +sub decorate_result ($) +{ + my $result = shift; + return $result unless $cfg{"color-tests"}; + my %color_for_result = + ( + "ERROR" => 'mgn', + "PASS" => 'grn', + "XPASS" => 'red', + "FAIL" => 'red', + "XFAIL" => 'lgn', + "SKIP" => 'blu', + ); + if (my $color = $color_for_result{$result}) + { + return colored ($color, $result); + } + else + { + return $result; # Don't colorize unknown stuff. + } +} + +sub report ($;$) +{ + my ($msg, $result, $explanation) = (undef, @_); + if ($result =~ /^(?:X?(?:PASS|FAIL)|SKIP|ERROR)/) + { + $msg = ": $test_script_name"; + add_test_result $result; + } + elsif ($result eq "#") + { + $msg = " $test_script_name:"; + } + else + { + die "$ME: INTERNAL ERROR"; # NOTREACHED + } + $msg .= " $explanation" if defined $explanation; + $msg .= "\n"; + # Output on console might be colorized. + print OLDOUT decorate_result ($result) . $msg; + # Log the result in the log file too, to help debugging (this is + # especially true when said result is a TAP error or "Bail out!"). + print $result . $msg; +} + +sub testsuite_error ($) +{ + report "ERROR", "- $_[0]"; +} + +sub handle_tap_result ($) +{ + $testno++; + my $result_obj = shift; + + my $test_result = stringify_result_obj $result_obj; + my $string = $result_obj->number; + + my $description = $result_obj->description; + $string .= " $description" + unless is_null_string $description; + + if ($plan_seen == LATE_PLAN) + { + $string .= " # AFTER LATE PLAN"; + } + elsif ($result_obj->is_unplanned) + { + $string .= " # UNPLANNED"; + } + elsif ($result_obj->number != $testno) + { + $string .= " # OUT-OF-ORDER (expecting $testno)"; + } + elsif (my $directive = $result_obj->directive) + { + $string .= " # $directive"; + my $explanation = $result_obj->explanation; + $string .= " $explanation" + unless is_null_string $explanation; + } + + report $test_result, $string; +} + +sub handle_tap_plan ($) +{ + my $plan = shift; + if ($plan_seen) + { + # Error, only one plan per stream is acceptable. + testsuite_error "multiple test plans"; + return; + } + # The TAP plan can come before or after *all* the TAP results; we speak + # respectively of an "early" or a "late" plan. If we see the plan line + # after at least one TAP result has been seen, assume we have a late + # plan; in this case, any further test result seen after the plan will + # be flagged as an error. + $plan_seen = ($testno >= 1 ? LATE_PLAN : EARLY_PLAN); + # If $testno > 0, we have an error ("too many tests run") that will be + # automatically dealt with later, so don't worry about it here. If + # $plan_seen is true, we have an error due to a repeated plan, and that + # has already been dealt with above. Otherwise, we have a valid "plan + # with SKIP" specification, and should report it as a particular kind + # of SKIP result. + if ($plan->directive && $testno == 0) + { + my $explanation = is_null_string ($plan->explanation) ? + undef : "- " . $plan->explanation; + report "SKIP", $explanation; + } +} + +sub handle_tap_bailout ($) +{ + my ($bailout, $msg) = ($_[0], "Bail out!"); + $bailed_out = 1; + $msg .= " " . $bailout->explanation + unless is_null_string $bailout->explanation; + testsuite_error $msg; +} + +sub extract_tap_comment ($) +{ + my $line = shift; + if (index ($line, $diag_string) == 0) + { + # Strip leading '$diag_string' from '$line'. + $line = substr ($line, length ($diag_string)); + # And strip any leading and trailing whitespace left. + $line =~ s/(?:^\s*|\s*$)//g; + # Return what is left (if any). + return $line; + } + return ""; +} + +sub finish () +{ + write_test_results; + close LOG or die "$ME: closing $log_file: $!\n"; + exit 0; +} + +sub main (@) +{ + setup_io; + setup_parser @_; + + while (defined (my $cur = $parser->next)) + { + # Verbatim copy any input line into the log file. + print $cur->raw . "\n"; + # Parsing of TAP input should stop after a "Bail out!" directive. + next if $bailed_out; + + if ($cur->is_plan) + { + handle_tap_plan ($cur); + } + elsif ($cur->is_test) + { + handle_tap_result ($cur); + } + elsif ($cur->is_bailout) + { + handle_tap_bailout ($cur); + } + elsif ($cfg{comments}) + { + my $comment = extract_tap_comment ($cur->raw); + report "#", "$comment" if length $comment; + } + } + # A "Bail out!" directive should cause us to ignore any following TAP + # error, as well as a non-zero exit status from the TAP producer. + if (!$bailed_out) + { + if (!$plan_seen) + { + testsuite_error "missing test plan"; + } + elsif ($parser->tests_planned != $parser->tests_run) + { + my ($planned, $run) = ($parser->tests_planned, $parser->tests_run); + my $bad_amount = $run > $planned ? "many" : "few"; + testsuite_error (sprintf "too %s tests run (expected %d, got %d)", + $bad_amount, $planned, $run); + } + if (!$cfg{"ignore-exit"}) + { + my $msg = get_test_exit_message (); + testsuite_error $msg if $msg; + } + } + finish; +} + +# ----------- # +# Main code. # +# ----------- # + +main @ARGV; + +# Local Variables: +# perl-indent-level: 2 +# perl-continued-statement-offset: 2 +# perl-continued-brace-offset: 0 +# perl-brace-offset: 0 +# perl-brace-imaginary-offset: 0 +# perl-label-offset: -2 +# cperl-indent-level: 2 +# cperl-brace-offset: 0 +# cperl-continued-brace-offset: 0 +# cperl-label-offset: -2 +# cperl-extra-newline-before-brace: t +# cperl-merge-trailing-else: nil +# cperl-continued-statement-offset: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "my $VERSION = " +# time-stamp-format: "'%:y-%02m-%02d.%02H'" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/doc/automake.texi b/doc/automake.texi index cd33ad7c3..74a410177 100644 --- a/doc/automake.texi +++ b/doc/automake.texi @@ -9742,7 +9742,7 @@ echo ok 1 # Exit with error, even if all the tests have been successful. exit 7 -% @kbd{cp @var{PREFIX}/share/automake-@var{APIVERSION}/tap-driver.pl .} +% @kbd{cp @var{PREFIX}/share/automake-@var{APIVERSION}/tap-driver.sh .} % @kbd{autoreconf -vi && ./configure && make check} ... PASS: foo.test 1 - Swallows fly diff --git a/gen-testsuite-part b/gen-testsuite-part index 3bd5c9f01..5bd1b4e96 100755 --- a/gen-testsuite-part +++ b/gen-testsuite-part @@ -241,20 +241,6 @@ my %test_generators = shell_setup_code => 'am_test_prefer_config_shell=yes', }, - # - # Tests on tap support should be run with both the perl and awk - # implementations of the TAP driver (they run with the awk one - # by default). - # - perl_tap_driver => - { - line_matcher => - qr<(?:\bfetch_tap_driver\b|[\s/]tap-setup\.sh\b)>, - line_rejecter => - qr/\bam_tap_implementation=/, - shell_setup_code => - 'am_tap_implementation=perl', - }, ); #-------------------------------------------------------------------------- diff --git a/lib/Makefile.inc b/lib/Makefile.inc index d1971f55f..8eed3038e 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc @@ -40,8 +40,7 @@ dist_script_DATA = \ %D%/py-compile \ %D%/ar-lib \ %D%/test-driver \ - %D%/tap-driver.sh \ - %D%/tap-driver.pl + %D%/tap-driver.sh install-data-hook: @$(POST_INSTALL) diff --git a/lib/tap-driver.pl b/lib/tap-driver.pl deleted file mode 100755 index aca65fe44..000000000 --- a/lib/tap-driver.pl +++ /dev/null @@ -1,564 +0,0 @@ -#! /usr/bin/env perl -# Copyright (C) 2011-2013 Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2, or (at your option) -# any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# This file is maintained in Automake, please report -# bugs to or send patches to -# . - -# ---------------------------------- # -# Imports, static data, and setup. # -# ---------------------------------- # - -use warnings FATAL => 'all'; -use strict; -use Getopt::Long (); -use TAP::Parser; - -my $VERSION = '2012-02-01.19'; # UTC - -my $ME = "tap-driver.pl"; - -my $USAGE = <<'END'; -Usage: - tap-driver --test-name=NAME --log-file=PATH --trs-file=PATH - [--expect-failure={yes|no}] [--color-tests={yes|no}] - [--enable-hard-errors={yes|no}] [--ignore-exit] - [--diagnostic-string=STRING] [--merge|--no-merge] - [--comments|--no-comments] [--] TEST-COMMAND -The `--test-name', `--log-file' and `--trs-file' options are mandatory. -END - -my $HELP = "$ME: TAP-aware test driver for Automake testsuite harness." . - "\n" . $USAGE; - -# Keep this in sync with `lib/am/check.am:$(am__tty_colors)'. -my %COLOR = ( - red => "\e[0;31m", - grn => "\e[0;32m", - lgn => "\e[1;32m", - blu => "\e[1;34m", - mgn => "\e[0;35m", - brg => "\e[1m", - std => "\e[m", -); - -# It's important that NO_PLAN evaluates "false" as a boolean. -use constant NO_PLAN => 0; -use constant EARLY_PLAN => 1; -use constant LATE_PLAN => 2; - -# ------------------- # -# Global variables. # -# ------------------- # - -my $testno = 0; # Number of test results seen so far. -my $bailed_out = 0; # Whether a "Bail out!" directive has been seen. -my $parser; # TAP parser object (will be initialized later). - -# Whether the TAP plan has been seen or not, and if yes, which kind -# it is ("early" is seen before any test result, "late" otherwise). -my $plan_seen = NO_PLAN; - -# ----------------- # -# Option parsing. # -# ----------------- # - -my %cfg = ( - "color-tests" => 0, - "expect-failure" => 0, - "merge" => 0, - "comments" => 0, - "ignore-exit" => 0, -); - -my $test_script_name = undef; -my $log_file = undef; -my $trs_file = undef; -my $diag_string = "#"; - -Getopt::Long::GetOptions - ( - 'help' => sub { print $HELP; exit 0; }, - 'version' => sub { print "$ME $VERSION\n"; exit 0; }, - 'test-name=s' => \$test_script_name, - 'log-file=s' => \$log_file, - 'trs-file=s' => \$trs_file, - 'color-tests=s' => \&bool_opt, - 'expect-failure=s' => \&bool_opt, - 'enable-hard-errors=s' => sub {}, # No-op. - 'diagnostic-string=s' => \$diag_string, - 'comments' => sub { $cfg{"comments"} = 1; }, - 'no-comments' => sub { $cfg{"comments"} = 0; }, - 'merge' => sub { $cfg{"merge"} = 1; }, - 'no-merge' => sub { $cfg{"merge"} = 0; }, - 'ignore-exit' => sub { $cfg{"ignore-exit"} = 1; }, - ) or exit 1; - -# ------------- # -# Prototypes. # -# ------------- # - -sub add_test_result ($); -sub bool_opt ($$); -sub colored ($$); -sub copy_in_global_log (); -sub decorate_result ($); -sub extract_tap_comment ($); -sub finish (); -sub get_global_test_result (); -sub get_test_exit_message (); -sub get_test_results (); -sub handle_tap_bailout ($); -sub handle_tap_plan ($); -sub handle_tap_result ($); -sub is_null_string ($); -sub main (@); -sub must_recheck (); -sub report ($;$); -sub setup_io (); -sub setup_parser (@); -sub stringify_result_obj ($); -sub testsuite_error ($); -sub trap_perl_warnings_and_errors (); -sub write_test_results (); -sub yn ($); - -# -------------- # -# Subroutines. # -# -------------- # - -sub bool_opt ($$) -{ - my ($opt, $val) = @_; - if ($val =~ /^(?:y|yes)\z/i) - { - $cfg{$opt} = 1; - } - elsif ($val =~ /^(?:n|no)\z/i) - { - $cfg{$opt} = 0; - } - else - { - die "$ME: invalid argument '$val' for option '$opt'\n"; - } -} - -# If the given string is undefined or empty, return true, otherwise -# return false. This function is useful to avoid pitfalls like: -# if ($message) { print "$message\n"; } -# which wouldn't print anything if $message is the literal "0". -sub is_null_string ($) -{ - my $str = shift; - return ! (defined $str and length $str); -} - -# Convert a boolean to a "yes"/"no" string. -sub yn ($) -{ - my $bool = shift; - return $bool ? "yes" : "no"; -} - -TEST_RESULTS : -{ - my (@test_results_list, %test_results_seen); - - sub add_test_result ($) - { - my $res = shift; - push @test_results_list, $res; - $test_results_seen{$res} = 1; - } - - sub get_test_results () - { - return @test_results_list; - } - - # Whether the test script should be re-run by "make recheck". - sub must_recheck () - { - return grep { !/^(?:XFAIL|PASS|SKIP)$/ } (keys %test_results_seen); - } - - # Whether the content of the log file associated to this test should - # be copied into the "global" test-suite.log. - sub copy_in_global_log () - { - return grep { not $_ eq "PASS" } (keys %test_results_seen); - } - - # FIXME: this can certainly be improved ... - sub get_global_test_result () - { - return "ERROR" - if $test_results_seen{"ERROR"}; - return "FAIL" - if $test_results_seen{"FAIL"} || $test_results_seen{"XPASS"}; - return "SKIP" - if scalar keys %test_results_seen == 1 && $test_results_seen{"SKIP"}; - return "PASS"; - } - -} - -sub write_test_results () -{ - open RES, ">", $trs_file or die "$ME: opening $trs_file: $!\n"; - print RES ":global-test-result: " . get_global_test_result . "\n"; - print RES ":recheck: " . yn (must_recheck) . "\n"; - print RES ":copy-in-global-log: " . yn (copy_in_global_log) . "\n"; - foreach my $result (get_test_results) - { - print RES ":test-result: $result\n"; - } - close RES or die "$ME: closing $trs_file: $!\n"; -} - -sub trap_perl_warnings_and_errors () -{ - $SIG{__WARN__} = $SIG{__DIE__} = sub - { - # Be sure to send the warning/error message to the original stderr - # (presumably the console), not into the log file. - open STDERR, ">&OLDERR"; - die @_; - } -} - -sub setup_io () -{ - # Redirect stderr and stdout to a temporary log file. Save the - # original stdout stream, since we need it to print testsuite - # progress output. Save original stderr stream, so that we can - # redirect warning and error messages from perl there. - open LOG, ">", $log_file or die "$ME: opening $log_file: $!\n"; - open OLDOUT, ">&STDOUT" or die "$ME: duplicating stdout: $!\n"; - open OLDERR, ">&STDERR" or die "$ME: duplicating stdout: $!\n"; - *OLDERR = *OLDERR; # To pacify a "used only once" warning. - trap_perl_warnings_and_errors; - open STDOUT, ">&LOG" or die "$ME: redirecting stdout: $!\n"; - open STDERR, ">&LOG" or die "$ME: redirecting stderr: $!\n"; -} - -sub setup_parser (@) -{ - local $@ = ''; - eval { $parser = TAP::Parser->new ({exec => \@_, merge => $cfg{merge}}) }; - if ($@ ne '') - { - # Don't use the error message in $@ as set by TAP::Parser, since - # currently it's both too generic (at the point of being basically - # useless) and quite long. - report "ERROR", "- couldn't execute test script"; - finish; - } -} - -sub get_test_exit_message () -{ - my $wstatus = $parser->wait; - # Watch out for possible internal errors. - die "$ME: couldn't get the exit status of the TAP producer" - unless defined $wstatus; - # Return an undefined value if the producer exited with success. - return unless $wstatus; - # Otherwise, determine whether it exited with error or was terminated - # by a signal. - use POSIX qw (WIFEXITED WEXITSTATUS WIFSIGNALED WTERMSIG); - if (WIFEXITED ($wstatus)) - { - return sprintf "exited with status %d", WEXITSTATUS ($wstatus); - } - elsif (WIFSIGNALED ($wstatus)) - { - return sprintf "terminated by signal %d", WTERMSIG ($wstatus); - } - else - { - return "terminated abnormally"; - } -} - -sub stringify_result_obj ($) -{ - my $result_obj = shift; - my $COOKED_PASS = $cfg{"expect-failure"} ? "XPASS": "PASS"; - my $COOKED_FAIL = $cfg{"expect-failure"} ? "XFAIL": "FAIL"; - if ($result_obj->is_unplanned || $result_obj->number != $testno) - { - return "ERROR"; - } - elsif ($plan_seen == LATE_PLAN) - { - return "ERROR"; - } - elsif (!$result_obj->directive) - { - return $result_obj->is_ok ? $COOKED_PASS: $COOKED_FAIL; - } - elsif ($result_obj->has_todo) - { - return $result_obj->is_actual_ok ? "XPASS" : "XFAIL"; - } - elsif ($result_obj->has_skip) - { - return $result_obj->is_ok ? "SKIP" : $COOKED_FAIL; - } - die "$ME: INTERNAL ERROR"; # NOTREACHED -} - -sub colored ($$) -{ - my ($color_name, $text) = @_; - return $COLOR{$color_name} . $text . $COLOR{'std'}; -} - -sub decorate_result ($) -{ - my $result = shift; - return $result unless $cfg{"color-tests"}; - my %color_for_result = - ( - "ERROR" => 'mgn', - "PASS" => 'grn', - "XPASS" => 'red', - "FAIL" => 'red', - "XFAIL" => 'lgn', - "SKIP" => 'blu', - ); - if (my $color = $color_for_result{$result}) - { - return colored ($color, $result); - } - else - { - return $result; # Don't colorize unknown stuff. - } -} - -sub report ($;$) -{ - my ($msg, $result, $explanation) = (undef, @_); - if ($result =~ /^(?:X?(?:PASS|FAIL)|SKIP|ERROR)/) - { - $msg = ": $test_script_name"; - add_test_result $result; - } - elsif ($result eq "#") - { - $msg = " $test_script_name:"; - } - else - { - die "$ME: INTERNAL ERROR"; # NOTREACHED - } - $msg .= " $explanation" if defined $explanation; - $msg .= "\n"; - # Output on console might be colorized. - print OLDOUT decorate_result ($result) . $msg; - # Log the result in the log file too, to help debugging (this is - # especially true when said result is a TAP error or "Bail out!"). - print $result . $msg; -} - -sub testsuite_error ($) -{ - report "ERROR", "- $_[0]"; -} - -sub handle_tap_result ($) -{ - $testno++; - my $result_obj = shift; - - my $test_result = stringify_result_obj $result_obj; - my $string = $result_obj->number; - - my $description = $result_obj->description; - $string .= " $description" - unless is_null_string $description; - - if ($plan_seen == LATE_PLAN) - { - $string .= " # AFTER LATE PLAN"; - } - elsif ($result_obj->is_unplanned) - { - $string .= " # UNPLANNED"; - } - elsif ($result_obj->number != $testno) - { - $string .= " # OUT-OF-ORDER (expecting $testno)"; - } - elsif (my $directive = $result_obj->directive) - { - $string .= " # $directive"; - my $explanation = $result_obj->explanation; - $string .= " $explanation" - unless is_null_string $explanation; - } - - report $test_result, $string; -} - -sub handle_tap_plan ($) -{ - my $plan = shift; - if ($plan_seen) - { - # Error, only one plan per stream is acceptable. - testsuite_error "multiple test plans"; - return; - } - # The TAP plan can come before or after *all* the TAP results; we speak - # respectively of an "early" or a "late" plan. If we see the plan line - # after at least one TAP result has been seen, assume we have a late - # plan; in this case, any further test result seen after the plan will - # be flagged as an error. - $plan_seen = ($testno >= 1 ? LATE_PLAN : EARLY_PLAN); - # If $testno > 0, we have an error ("too many tests run") that will be - # automatically dealt with later, so don't worry about it here. If - # $plan_seen is true, we have an error due to a repeated plan, and that - # has already been dealt with above. Otherwise, we have a valid "plan - # with SKIP" specification, and should report it as a particular kind - # of SKIP result. - if ($plan->directive && $testno == 0) - { - my $explanation = is_null_string ($plan->explanation) ? - undef : "- " . $plan->explanation; - report "SKIP", $explanation; - } -} - -sub handle_tap_bailout ($) -{ - my ($bailout, $msg) = ($_[0], "Bail out!"); - $bailed_out = 1; - $msg .= " " . $bailout->explanation - unless is_null_string $bailout->explanation; - testsuite_error $msg; -} - -sub extract_tap_comment ($) -{ - my $line = shift; - if (index ($line, $diag_string) == 0) - { - # Strip leading `$diag_string' from `$line'. - $line = substr ($line, length ($diag_string)); - # And strip any leading and trailing whitespace left. - $line =~ s/(?:^\s*|\s*$)//g; - # Return what is left (if any). - return $line; - } - return ""; -} - -sub finish () -{ - write_test_results; - close LOG or die "$ME: closing $log_file: $!\n"; - exit 0; -} - -sub main (@) -{ - setup_io; - setup_parser @_; - - while (defined (my $cur = $parser->next)) - { - # Verbatim copy any input line into the log file. - print $cur->raw . "\n"; - # Parsing of TAP input should stop after a "Bail out!" directive. - next if $bailed_out; - - if ($cur->is_plan) - { - handle_tap_plan ($cur); - } - elsif ($cur->is_test) - { - handle_tap_result ($cur); - } - elsif ($cur->is_bailout) - { - handle_tap_bailout ($cur); - } - elsif ($cfg{comments}) - { - my $comment = extract_tap_comment ($cur->raw); - report "#", "$comment" if length $comment; - } - } - # A "Bail out!" directive should cause us to ignore any following TAP - # error, as well as a non-zero exit status from the TAP producer. - if (!$bailed_out) - { - if (!$plan_seen) - { - testsuite_error "missing test plan"; - } - elsif ($parser->tests_planned != $parser->tests_run) - { - my ($planned, $run) = ($parser->tests_planned, $parser->tests_run); - my $bad_amount = $run > $planned ? "many" : "few"; - testsuite_error (sprintf "too %s tests run (expected %d, got %d)", - $bad_amount, $planned, $run); - } - if (!$cfg{"ignore-exit"}) - { - my $msg = get_test_exit_message (); - testsuite_error $msg if $msg; - } - } - finish; -} - -# ----------- # -# Main code. # -# ----------- # - -main @ARGV; - -# Local Variables: -# perl-indent-level: 2 -# perl-continued-statement-offset: 2 -# perl-continued-brace-offset: 0 -# perl-brace-offset: 0 -# perl-brace-imaginary-offset: 0 -# perl-label-offset: -2 -# cperl-indent-level: 2 -# cperl-brace-offset: 0 -# cperl-continued-brace-offset: 0 -# cperl-label-offset: -2 -# cperl-extra-newline-before-brace: t -# cperl-merge-trailing-else: nil -# cperl-continued-statement-offset: 2 -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "my $VERSION = " -# time-stamp-format: "'%:y-%02m-%02d.%02H'" -# time-stamp-time-zone: "UTC" -# time-stamp-end: "; # UTC" -# End: diff --git a/t/ax/am-test-lib.sh b/t/ax/am-test-lib.sh index 26e58ef04..cee5c8da8 100644 --- a/t/ax/am-test-lib.sh +++ b/t/ax/am-test-lib.sh @@ -583,17 +583,31 @@ count_test_results () # of /bin/sh. get_shell_script () { - test ! -f "$1" || rm -f "$1" || return 99 + am_source=$1 am_target=${2-$1} + test ! -f "$am_target" || rm -f "$am_target" || return 99 if test x"$am_test_prefer_config_shell" = x"yes"; then - sed "1s|#!.*|#! $SHELL|" "$am_scriptdir/$1" > "$1" \ - && chmod a+x "$1" \ + sed "1s|#!.*|#! $SHELL|" "$am_scriptdir/$am_source" > "$am_target" \ + && chmod a+x "$am_target" \ || return 99 else - cp -f "$am_scriptdir/$1" . || return 99 + cp -f "$am_scriptdir/$am_source" "$am_target" || return 99 fi - sed 10q "$1" # For debugging. + sed 10q "$am_target" # For debugging. + unset am_target am_source } +# fetch_tap_driver +# ---------------- +# Fetch the Automake-provided TAP driver from the 'lib/' directory into +# the current directory, and edit its shebang line so that it will be +# run with the proper shell. +fetch_tap_driver () +{ + AM_TAP_AWK=$AWK; export AM_TAP_AWK + get_shell_script tap-driver.sh tap-driver +} + + # require_xsi SHELL # ----------------- # Skip the test if the given shell fails to support common XSI constructs. @@ -612,35 +626,6 @@ xsi_shell_code=' && eval '\''test $(( 1 + 1 )) -eq 2 \ && test "${#_lt_dummy}" -eq 5'\' -# fetch_tap_driver -# ---------------- -# Fetch the Automake-provided TAP driver from the 'lib/' directory into -# the current directory, and edit its shebang line so that it will be -# run with the perl interpreter determined at configure time. -fetch_tap_driver () -{ - # TODO: we should devise a way to make the shell TAP driver tested also - # TODO: with /bin/sh, for better coverage. - case $am_tap_implementation in - # Extra quoting required to avoid maintainer-check spurious failures. - 'perl') - $PERL -MTAP::Parser -e 1 \ - || skip_all_ "cannot import TAP::Parser perl module" - sed "1s|#!.*|#! $PERL -w|" "$am_scriptdir"/tap-driver.pl >tap-driver - ;; - shell) - AM_TAP_AWK=$AWK; export AM_TAP_AWK - sed "1s|#!.*|#! $SHELL|" "$am_scriptdir"/tap-driver.sh >tap-driver - ;; - *) - fatal_ "invalid \$am_tap_implementation '$am_tap_implementation'" ;; - esac \ - && chmod a+x tap-driver \ - || framework_failure_ "couldn't fetch $am_tap_implementation TAP driver" - sed 10q tap-driver # For debugging. -} -am_tap_implementation=${am_tap_implementation-shell} - # $PYTHON and support for PEP-3147. Needed to check our python-related # install rules. python_has_pep3147 () diff --git a/t/tap-bad-prog.tap b/t/tap-bad-prog.tap index 490cd4a78..9225c38ed 100644 --- a/t/tap-bad-prog.tap +++ b/t/tap-bad-prog.tap @@ -73,35 +73,13 @@ else fi # Check that no spurious test result is reported. This is lower-priority -# (and in fact the check currently fails for our awk-based driver). -directive= -if test $am_tap_implementation = shell; then - directive=TODO -else - # Older versions of IPC::Open3 (e.g., version 1.05 on perl 5.12.4 or - # version 1.0103 on perl 5.6.2) fail to properly trap errors in exec(2) - # calls in the child process; hence, the TAP driver cannot be properly - # informed of such error. - if $PERL -w -e ' - use IPC::Open3 qw/open3/; - $@ = ""; - eval { open3(*STDIN, *STDOUT, *STDERR, "am--no-such-command") }; - $@ =~ m/\bopen3:.*am--no-such-command/ - or die "Bad \$@ value: \"$@\"\n"; - '; then - : # OK. IPC::Open3 should be good enough. - else - for s in '"missing plan" message' 'results'; do - skip_ -r "IPC::Open3 not good enough" "no spurious $s" - done - exit 0 - fi -fi +# (and in fact the check currently fails). command_ok_ 'no spurious "missing plan" message' \ - -D "$directive" -- not grep 'missing.* plan' stdout + -D TODO -- not grep 'missing.* plan' stdout + command_ok_ 'no spurious results' \ - -D "$directive" -r 'still get "missing plan"' \ + -D TODO -r 'still get "missing plan"' \ count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3 : diff --git a/t/tap-bailout-leading-space.sh b/t/tap-bailout-leading-space.sh index 120ae0340..7a55a1fba 100644 --- a/t/tap-bailout-leading-space.sh +++ b/t/tap-bailout-leading-space.sh @@ -14,15 +14,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Older versions of prove and TAP::Harness (e.g., 3.17) didn't recognize -# a "Bail out!" directive that was preceded by whitespace, but more modern -# versions (e.g., 3.23) do. So we leave this behaviour undefined for the -# perl implementation of the Automake TAP driver, but expect the latter, -# "more modern" behaviour in our awk TAP driver. +# A "Bail out!" directive that is preceded by whitespace should still +# be recognized. -am_tap_implementation=shell . test-init.sh - . tap-setup.sh cat > a.test < "all"; use strict; - use TAP::Parser; - my $parser = TAP::Parser->new({tap => "1..1\n" . "ok 0\n"}); - my $result = $parser->next; - $result->is_plan or die "first line is not TAP plan"; - $result = $parser->next; - $result->is_test or die "second line is not TAP test result"; - my $testno = $result->number; - $parser->next and die "unexpected further TAP stream"; - exit ($testno == 0 ? 0 : 77); - '; then - : # Nothing to do. - elif test $? -eq 77; then - skip_ 'TAP::Parser bug: test number 0 gets relabelled as 1' - else - fatal_ "error analyzing TAP::Parser module for bugs" - fi -fi - . tap-setup.sh cat > a.test <> configure.ac << END AM_CONDITIONAL([COND1], [:]) @@ -40,7 +40,7 @@ $AUTOCONF cat > Makefile.am << 'END' TESTS = foo bar.test baz.sh -EXTRA_DIST = $(TESTS) tap-driver.pl trivial-test-driver +EXTRA_DIST = $(TESTS) tap-driver.sh trivial-test-driver TEST_EXTENSIONS = .test .sh LOG_DRIVER = SH_LOG_DRIVER = $(tap_rulez) @@ -49,7 +49,7 @@ LOG_DRIVER += @my_LOG_DRIVER@ if COND2 tap_rulez = false else !COND2 -tap_rulez = $(PERL) $(srcdir)/tap-driver.pl +tap_rulez = $(PERL) $(srcdir)/tap-driver.sh endif !COND2 endif COND1 END -- cgit v1.2.1 From c7e2fb2e73b2d1b904feef0ed5e87ce05a41c119 Mon Sep 17 00:00:00 2001 From: Stefano Lattarini Date: Tue, 24 Dec 2013 17:45:18 +0100 Subject: testsuite harness: report test exit status in log file The exit status of a test should be reported in the test logs, so that one can see at a glance whether the test has succeeded or failed, without having to look also into the corresponding .trs file. This fixes automake bug#11814. * lib/test-driver: Also report the test script exit status in the test log (as the last line). * t/check-exit-status-reported.sh: Test this new behaviour. * t/list-of-tests.mk: Add the new test. * t/ax/test-lib.sh( am_exit_trap): No longer log the test exit status; this has been made redundant by the change to 'test-driver'. While at it, fix an imperfect quoting. Signed-off-by: Stefano Lattarini --- NEWS | 6 +++ lib/test-driver | 13 +++++- t/ax/test-lib.sh | 3 +- t/list-of-tests.mk | 1 + t/parallel-tests-exit-status-reported.sh | 68 ++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 t/parallel-tests-exit-status-reported.sh diff --git a/NEWS b/NEWS index 5cc001977..61ed194cb 100644 --- a/NEWS +++ b/NEWS @@ -73,6 +73,12 @@ New in 1.15: (3) the "set -f" and "set +f" shell commands work, and, respectively, disable and enable shell globbing. +* Automake-generated testsuites: + + - The default test-driver used by the Automake-generates testsuites now + append the result and exit status of each "plain" test to the associated + log file (automake bug#118149). + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ New in 1.14.1: diff --git a/lib/test-driver b/lib/test-driver index d30605660..110eec409 100755 --- a/lib/test-driver +++ b/lib/test-driver @@ -106,11 +106,14 @@ trap "st=143; $do_exit" 15 # Test script is run here. "$@" >$log_file 2>&1 estatus=$? + if test $enable_hard_errors = no && test $estatus -eq 99; then - estatus=1 + tweaked_estatus=1 +else + tweaked_estatus=$estatus fi -case $estatus:$expect_failure in +case $tweaked_estatus:$expect_failure in 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; 0:*) col=$grn res=PASS recheck=no gcopy=no;; 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; @@ -119,6 +122,12 @@ case $estatus:$expect_failure in *:*) col=$red res=FAIL recheck=yes gcopy=yes;; esac +# Report the test outcome and exit status in the logs, so that one can +# know whether the test passed or failed simply by looking at the '.log' +# file, without the need of also peaking into the corresponding '.trs' +# file (automake bug#11814). +echo "$res $test_name (exit status: $estatus)" >>$log_file + # Report outcome to console. echo "${col}${res}${std}: $test_name" diff --git a/t/ax/test-lib.sh b/t/ax/test-lib.sh index aa015d68d..b8c72533c 100644 --- a/t/ax/test-lib.sh +++ b/t/ax/test-lib.sh @@ -254,7 +254,7 @@ am_exit_trap () # behaviour, while from time to time useful to developers, is not # meant to be enabled by default, as it could cause spurious failures # in the wild. Thus it will be enabled only when the variable - # "am_explicit_skips" is set to a "true" value. + # 'am_explicit_skips' is set to a "true" value. case $am_explicit_skips in [yY]|[yY]es|1) if test $exit_status -eq 77 && test $am__test_skipped != yes; then @@ -266,7 +266,6 @@ am_exit_trap () fi am_keeping_testdirs || rm_rf_ $am_test_subdir set +x - echo "$me: exit $exit_status" # Spurious escaping to ensure we do not call our "exit" alias. \exit $exit_status } diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk index 462497ee3..ba6578903 100644 --- a/t/list-of-tests.mk +++ b/t/list-of-tests.mk @@ -765,6 +765,7 @@ t/parallel-tests-basics.sh \ t/parallel-tests-concurrency.sh \ t/parallel-tests-concurrency-2.sh \ t/parallel-tests-empty.sh \ +t/parallel-tests-exit-status-reported.sh \ t/parallel-tests-generated-and-distributed.sh \ t/parallel-tests-recheck.sh \ t/parallel-tests-trailing-whitespace.sh \ diff --git a/t/parallel-tests-exit-status-reported.sh b/t/parallel-tests-exit-status-reported.sh new file mode 100644 index 000000000..d40f1f58a --- /dev/null +++ b/t/parallel-tests-exit-status-reported.sh @@ -0,0 +1,68 @@ +#! /bin/sh +# Copyright (C) 2013 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# The exit status of a test should be reported in the test logs, so +# that one can see at a glance whether the test has succeeded or failed, +# without having to look also into the corresponding .trs file. +# See automake bug#11814. + +. test-init.sh + +echo AC_OUTPUT >> configure.ac + +echo XFAIL_TESTS = t3.test t00.test > Makefile.am +echo TESTS = t00.test >> Makefile.am +for s in 0 1 2 3 5 77 78 99 100 126 127; do + echo "TESTS += t${s}.test" >> Makefile.am + cat > t${s}.test < Date: Tue, 24 Dec 2013 21:16:23 +0100 Subject: docs: drop a few obsolescent FIXME/TODO comments, and associated text Signed-off-by: Stefano Lattarini --- doc/automake.texi | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/doc/automake.texi b/doc/automake.texi index cd33ad7c3..09aa7d173 100644 --- a/doc/automake.texi +++ b/doc/automake.texi @@ -1900,9 +1900,6 @@ It is customary to make the first line of @file{Makefile.am} read: ## Process this file with automake to produce Makefile.in @end example -@c FIXME discuss putting a copyright into Makefile.am here? I would but -@c I don't know quite what to say. - @c FIXME document customary ordering of Makefile.am here! @@ -8821,9 +8818,6 @@ error} happens when e.g., the set-up of a test case scenario fails, or when some other unexpected or highly undesirable condition is encountered (for example, the program under test experiences a segmentation fault). -@emph{TODO}: Links to other test harnesses (esp. those sharing our -terminology)? - @node Simple Tests @section Simple Tests @@ -8956,8 +8950,6 @@ flag on file descriptors opened with the @command{exec} builtin, thus rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;} ineffectual. This issue also affects some Bourne shells, such as the HP-UX's @command{/bin/sh}, -@c FIXME: should we offer a link to the relevant discussions on the -@c bug-autoconf list? @c Keep in sync with tests-environment-backcompat.sh @example @@ -9017,7 +9009,6 @@ The serial test harness is enabled by the Automake option @option{serial-tests}. It operates by simply running the tests serially, one at the time, without any I/O redirection. It's up to the user to implement logging of tests' output, if that's requited or desired. -@c TODO: give an example of how this can be done. For historical and implementation reasons, the @code{AM_TESTS_ENVIRONMENT} variable is @emph{not} supported by this harness (it will be silently @@ -9086,8 +9077,6 @@ to a per-test log file, so that parallel execution does not produce intermingled output. The output from failed tests is collected in the @file{test-suite.log} file. If the variable @samp{VERBOSE} is set, this file is output after the summary. -@c FIXME: we should be clearer about what we mean exactly here ... -For best results, the tests should be verbose by default now. @vindex TEST_EXTENSIONS @vindex TEST_LOGS @@ -9596,12 +9585,6 @@ other end, if you are using a known and widespread test protocol with well-established implementations, being consistent with those implementations' output might be a good idea too. -@c TODO: Give an example, maybe inspired to py.test-style output. -@c TODO: That is a good idea because it shows a test driver that allows -@c TODO: for different levels of verbosity in the progress output (could -@c TODO: be implemented either using a driver cmdline flag, or an -@c TODO: environment variable, or both). - @node Using the TAP test protocol @section Using the TAP test protocol @@ -10838,9 +10821,6 @@ they all have their serious drawbacks and limitations. That's why automake provides support for a more advanced and flexible way of obtaining quieter output from @command{make} (for most rules at least). -@c TODO: Maybe describe in brief the precedent set by the build system -@c of the Linux Kernel, from which Automake took inspiration ... Links? - To give the gist of what Automake can do in this respect, here is a simple comparison between a typical @command{make} output (where silent rules are disabled) and one with silent rules enabled: -- cgit v1.2.1