testsuite: yet more use of TAP, and related extensions

* tests/defs (show_): New function, display the contents of one or
more files on stdout, with bells & whistles (both for cosmetic and
practical reasons, the latter aimed at avoiding spurious TAP
diagnostic).
(AUTOMAKE_run): Extend and adapt to make it compatible with TAP
based tests.  Since we are at it, make its implementation more
namespace-safe, and improve its command-line interface.
(AUTOMAKE_fails): Adapt to the new command-line interface of
`AUTOMAKE_run'.
* tests/add-missing.test: Renamed ...
* tests/add-missing.tap: ... to this, and converted to the use
of the TAP protocol, and of the new `show_' function.
* tests/automake.test: Renamed ...
* tests/automake-cmdline.tap: ... to this, and converted to the
use of the TAP protocol.
* tests/getopt.test:  Adapt to the new command-line interface
of `AUTOMAKE_run'.
* tests/cond44.test: Likewise.
* tests/cond45.test: Likewise.
* tests/configure.test: Likewise.
* tests/license2.test: Likewise.
* tests/parallel-am.test: Likewise.
* tests/parallel-am3.test: Likewise.
* tests/tags2.test: Likewise.
* tests/werror3.test: Likewise.
* tests/werror4.test: Likewise.
* tests/amopts-variable-expansion.test: Likewise.
* tests/warnings-win-over-strictness.test: Likewise.
* tests/Makefile.am (TESTS): Update.
(TAP_LOG_DRIVER_FLAGS): Add `--merge', so that TAP diagnostic is
reported in the testsuite progress output.

19 files changed, 283 insertions, 134 deletions

diff --git a/ChangeLog b/ChangeLog
index bd67daba2..ad8c02394 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,38 @@
+2011-08-04  Stefano Lattarini  <stefano.lattarini@gmail.com>
+
+	testsuite: yet more use of TAP, and related extensions
+	* tests/defs (show_): New function, display the contents of one or
+	more files on stdout, with bells & whistles (both for cosmetic and
+	practical reasons, the latter aimed at avoiding spurious TAP
+	diagnostic).
+	(AUTOMAKE_run): Extend and adapt to make it compatible with TAP
+	based tests.  Since we are at it, make its implementation more
+	namespace-safe, and improve its command-line interface.
+	(AUTOMAKE_fails): Adapt to the new command-line interface of
+	`AUTOMAKE_run'.
+	* tests/add-missing.test: Renamed ...
+	* tests/add-missing.tap: ... to this, and converted to the use
+	of the TAP protocol, and of the new `show_' function.
+	* tests/automake.test: Renamed ...
+	* tests/automake-cmdline.tap: ... to this, and converted to the
+	use of the TAP protocol.
+	* tests/getopt.test:  Adapt to the new command-line interface
+	of `AUTOMAKE_run'.
+	* tests/cond44.test: Likewise.
+	* tests/cond45.test: Likewise.
+	* tests/configure.test: Likewise.
+	* tests/license2.test: Likewise.
+	* tests/parallel-am.test: Likewise.
+	* tests/parallel-am3.test: Likewise.
+	* tests/tags2.test: Likewise.
+	* tests/werror3.test: Likewise.
+	* tests/werror4.test: Likewise.
+	* tests/amopts-variable-expansion.test: Likewise.
+	* tests/warnings-win-over-strictness.test: Likewise.
+	* tests/Makefile.am (TESTS): Update.
+	(TAP_LOG_DRIVER_FLAGS): Add `--merge', so that TAP diagnostic is
+	reported in the testsuite progress output.
+
 2011-08-03  Stefano Lattarini  <stefano.lattarini@gmail.com>
 
 	maintcheck: fix more maintainer-check failures
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 939ed254e..0920e6b10 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -18,7 +18,7 @@
 
 TEST_EXTENSIONS = .test .tap
 TAP_LOG_DRIVER = $(PERL) $(top_srcdir)/lib/tap-driver
-TAP_LOG_DRIVER_FLAGS = --merge
+TAP_LOG_DRIVER_FLAGS = --merge --comments
 
 MAINTAINERCLEANFILES =
 EXTRA_DIST = ChangeLog-old
@@ -178,7 +178,7 @@ ac-output-old.tap \
 acsilent.test \
 acsubst.test \
 acsubst2.test \
-add-missing.test \
+add-missing.tap \
 all.test \
 all2.test \
 alloca.test \
@@ -226,7 +226,7 @@ autohdr2.test \
 autohdr3.test \
 autohdr4.test \
 autohdrdry.test \
-automake.test \
+automake-cmdline.tap \
 auxdir.test \
 auxdir2.test \
 auxdir3.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 59216598d..c6a313b65 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -300,7 +300,7 @@ top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 TEST_EXTENSIONS = .test .tap .ptest .instspc .depmod .shtst
 TAP_LOG_DRIVER = $(PERL) $(top_srcdir)/lib/tap-driver
-TAP_LOG_DRIVER_FLAGS = --merge
+TAP_LOG_DRIVER_FLAGS = --merge --comments
 MAINTAINERCLEANFILES = 
 EXTRA_DIST = ChangeLog-old gen-parallel-tests parallel-tests.sh \
 	instspc-tests.sh depmod-tests.sh gen-config-shell-tests \
@@ -454,7 +454,7 @@ ac-output-old.tap \
 acsilent.test \
 acsubst.test \
 acsubst2.test \
-add-missing.test \
+add-missing.tap \
 all.test \
 all2.test \
 alloca.test \
@@ -502,7 +502,7 @@ autohdr2.test \
 autohdr3.test \
 autohdr4.test \
 autohdrdry.test \
-automake.test \
+automake-cmdline.tap \
 auxdir.test \
 auxdir2.test \
 auxdir3.test \
diff --git a/tests/add-missing.test b/tests/add-missing.tap
index 9fd858d91..809b30f9f 100755
--- a/tests/add-missing.test
+++ b/tests/add-missing.tap
@@ -54,8 +54,8 @@ $ACLOCAL || framework_failure_ "cannot pre-compute aclocal.m4"
 rm -rf install-sh missing depcomp configure.in autom4te*.cache
 mv aclocal.m4 aclocal.stub
 
-cat configure.stub # For debugging.
-cat aclocal.stub   # Likewise.
+show_ configure.stub
+show_ aclocal.stub
 
 # This is hacky and ugly and complex, but allow us to organize our tests
 # below in a more "declarative fashion".  All in all, a good trade-off.
@@ -64,31 +64,32 @@ check_ ()
   set +x # Temporary disable shell traces to remove noise from log files.
   override=no
   run_aclocal=no
-  extra_file=
+  fetch_file=
   while test $# -gt 0; do
     case $1 in
       --override) override=yes;;
       --run-aclocal) run_aclocal=yes;;
-      --extra-file) extra_file=$2; shift;;
+      --fetch-file) fetch_file=$2; shift;;
       *) framework_failure_ "check_: invalid argument '$1'";;
     esac
     shift
   done
   mkdir testdir-generic
   cd testdir-generic
-  : > Makefile.am
+  # Use `echo > ...', not `: > ...', here and below, as Solaris 10 /bin/sh
+  # might not execute the latter (the Autoconf manual gives more details).
+  echo > Makefile.am
   if test $override = yes; then
-    : > configure.in
+    echo > configure.in
   else
     cp ../configure.stub configure.in
   fi
-  test -z "$extra_file" || cp ../"$extra_file" .
+  test -z "$fetch_file" || cp ../"$fetch_file" .
   # Read description of "test scenario" from standard input.
-  what=
-  line=
-  files='install-sh missing' # These are always installed by automake.
+  what= line= name= files=
   while read line; do
     case $line in
+      '== Name ==') what=NAME;;
       '== Makefile.am ==') what=Makefile.am;;
       '== configure.in ==') what=configure.in;;
       '== Files ==') what=LIST;;
@@ -97,26 +98,28 @@ check_ ()
       *)
         if test $what = LIST; then
           files="$files $line"
+        elif test $what = NAME; then
+          name=$line
         else
           printf '%s\n' "$line" >> "$what"
         fi
         ;;
     esac
   done
+  test -n "$name" || fatal_ "name of a test case not specified"
+  diag_ "Will run checks for $name."
   if test $run_aclocal = yes; then
-    $ACLOCAL
-    echo == aclocal.m4 ==
-    cat aclocal.m4
+    if $ACLOCAL; then
+      ok_ "[$name] aclocal.m4 rebuilt"
+      show_ aclocal.m4
+    else
+      not_ok_ "[$name] aclocal failure, aclocal.m4 not rebuilt"
+    fi
   else
     cp ../aclocal.stub aclocal.m4 
   fi
-  # For debugging.
-  if test -s Makefile.am; then
-    echo == Makefile.am ==
-    cat Makefile.am
-  fi
-  echo == configure.in ==
-  cat configure.in
+  show_ Makefile.am
+  show_ configure.in
   echo Expected files: $files
   mkdir "$build_aux"
   cd ..
@@ -126,62 +129,91 @@ check_ ()
   # Test once with `--copy', once without.
   for action in link copy; do
     case $action in
-      link) opts='--add-missing' test_linked='test -h';;
-      copy) opts='-a --copy' test_linked='test ! -h';;
-         *) Exit 99;; # Can't happen.
+      link) opts='--add-missing';;
+      copy) opts='-a --copy';;
+         *) fatal_ "invalid value '$action' for \$action";;
     esac
+    pfx="[$action $name]"
     cp -R testdir-generic testdir-$action
     cd testdir-$action
     # If the required auxiliary files are missing, and automake is
     # not told to install them, it should complain and error out,
     # and also give a useful suggestion.
-    AUTOMAKE_fails
+    AUTOMAKE_fails -d "$pfx missing files, automake fails"
     for f in $files; do 
-      grep "required file ['\`]$build_aux/$f' not found" stderr
+      command_ok_ \
+        "$pfx warn about missing file $f" \
+        grep "required file ['\`]$build_aux/$f' not found" stderr
       # Suggest the user to use `--add-missing'.
-      grep ".*--add-missing.* install .*$f" stderr
+      command_ok_ \
+        "$pfx suggest --add-missing for $f" \
+        grep ".*--add-missing.* install .*$f" stderr
     done
     # No files should be automatically installed by automake if it
     # is not told to.
-    ls "$build_aux" | grep . && Exit 1
-    $AUTOMAKE $opts 2>stderr || { cat stderr >&2; Exit 1; }
-    cat stderr >&2
+    if ls "$build_aux" | grep .; then r='not ok'; else r=ok; fi
+    result_ "$r" "$pfx no extra files installed"
+    AUTOMAKE_run -d "$pfx automake run successfully" -- $opts
     ls -l . $build_aux
     # The expected files should get installed correctly (e.g., no
     # broken symlinks).
     for f in $files; do
-      test -f $build_aux/$f
+      command_ok_ \
+        "$pfx file $f installed" \
+        test -f $build_aux/$f
     done
     # Automake should inform about which files it's installing.
     for f in $files; do 
-      grep ": installing ['\`]$build_aux/$f'$" stderr
+      command_ok_ \
+        "$pfx report installation of $f" \
+        grep ": installing ['\`]$build_aux/$f'$" stderr
     done
-    # Only the expected files should be installed.
-    for f in $files; do echo $f; done | sort > files.exp
+    # Only the expected files should be installed.  But automake always
+    # require `missing' and `install-sh', so account for them.
+    all_files="install-sh missing $files"
+    for f in $all_files; do echo $f; done | sort | uniq > files.exp
     (cd $build_aux && ls) | sort > files.got
-    cat files.exp
-    cat files.got
-    $am_diff files.exp files.got
+    show_ files.exp
+    show_ files.got
+    command_ok_ \
+      "$pfx all and only expected files installed" \
+      $am_diff files.exp files.got
     # The files should be copied by `--copy' and symlinked otherwise.
     for f in $files; do
-      $test_linked $build_aux/$f
+      if test -h $build_aux/$f; then
+        is_symlink=yes
+      else
+        is_symlink=no
+      fi
+      case $action,$is_symlink in
+        link,yes)     ok_ "$pfx $f has been symlinked"     ;;
+         link,no) not_ok_ "$pfx $f has not been symlinked" ;;
+        copy,yes) not_ok_ "$pfx $f has been symlinked"     ;;
+         copy,no)     ok_ "$pfx $f has not been symlinked" ;;
+               *)  fatal_ "invalid condition in case"      ;;
+      esac
     done
     # Now that the required auxiliary files have been installed, automake
     # should not complain anymore even if the `--add-missing' option is
     # not used.
-    $AUTOMAKE
+    AUTOMAKE_run -d "$pfx automake finds all added files"
     cd ..
-  done
+  done # for action in link copy
   rm -rf testdir-*
 }
 
-: %%% install-sh and missing %%%
+# Automake should always and unconditionally require the `missing'
+# and `install-sh' scripts.
 check_ <<'END'
-#% 'install-sh' and 'missing' should always get installed.
+== Name ==
+minimal
+== Files ==
+install-sh missing
 END
 
-: %%% depcomp with C %%%
 check_ <<'END'
+== Name ==
+depcomp/C
 == Files ==
 depcomp
 == configure.in ==
@@ -190,8 +222,9 @@ AC_PROG_CC
 bin_PROGRAMS = foo
 END
 
-: %%% depcomp with C++ %%%
 check_ <<'END'
+== Name ==
+depcomp/C++
 == Files ==
 depcomp
 == configure.in ==
@@ -201,8 +234,9 @@ bin_PROGRAMS = foo
 foo_SOURCES = foo.cc
 END
 
-: %%% compile script %%%
 check_ --run-aclocal <<'END'
+== Name ==
+compile
 == Files ==
 compile
 == configure.in ==
@@ -211,21 +245,23 @@ compile
 AM_PROG_CC_C_O
 END
 
-: %%% config.guess and config.sub %%%
-
-for macro in AC_CANONICAL_BUILD AC_CANONICAL_HOST \
-             AC_CANONICAL_TARGET AC_CANONICAL_SYSTEM; do
+# For config.guess and config.sub.
+for mach in build host target system; do
+  MACH=`echo "$mach" | LC_ALL=C tr '[a-z]' '[A-Z]'`
   check_ <<END
+== Name ==
+cfg-$mach
 == Files ==
 config.sub
 config.guess
 == configure.in ==
-$macro
+AC_CANONICAL_$MACH
 END
 done
 
-: %%% ylwrap with Lex %%%
 check_ <<'END'
+== Name ==
+ylwrap/Lex
 == Files ==
 ylwrap
 == configure.in ==
@@ -237,8 +273,9 @@ bin_PROGRAMS = foo
 foo_SOURCES = foo.l
 END
 
-: %%% ylwrap with Yacc %%%
 check_ <<'END'
+== Name ==
+ylwrap/Yacc
 == Files ==
 ylwrap
 == configure.in ==
@@ -250,18 +287,20 @@ bin_PROGRAMS = foo
 foo_SOURCES = foo.y
 END
 
-: %%%  texinfo.tex %%%
 echo '@setfilename foo.info' > foo.texi
-check_ --extra-file foo.texi <<'END'
+check_ --fetch-file foo.texi <<'END'
+== Name ==
+Texinfo
 == Files ==
 texinfo.tex
 == Makefile.am ==
 info_TEXINFOS = foo.texi
 END
 
-: %%% mdate-sh and texinfo.tex %%%
 echo '@include version.texi' >> foo.texi
-check_ --extra-file foo.texi <<'END'
+check_ --fetch-file foo.texi <<'END'
+== Name ==
+Texinfo/mdate-sh
 == Files ==
 mdate-sh
 texinfo.tex
@@ -271,8 +310,9 @@ END
 
 rm -f foo.texi
 
-: %%% elisp-compile %%%
 check_ <<'END'
+== Name ==
+elisp-comp
 == Files ==
 elisp-comp
 == configure.in ==
@@ -281,8 +321,9 @@ AM_PATH_LISPDIR
 lisp_LISP = foo.el
 END
 
-: %%% py-compile %%%
 check_ <<'END'
+== Name ==
+py-compile
 == Files ==
 py-compile
 == configure.in ==
@@ -291,8 +332,10 @@ AM_PATH_PYTHON
 python_PYTHON = foo.py
 END
 
-: %%% few unrelated auxiliary scripts together %%%
+# Try few unrelated auxiliary scripts together.
 check_ <<'END'
+== Name ==
+misc
 == Files ==
 py-compile
 depcomp
diff --git a/tests/amopts-variable-expansion.test b/tests/amopts-variable-expansion.test
index c11be8742..57512cc7c 100755
--- a/tests/amopts-variable-expansion.test
+++ b/tests/amopts-variable-expansion.test
@@ -44,7 +44,7 @@ install:
 END
 
 $ACLOCAL
-AUTOMAKE_run 0
+AUTOMAKE_run
 grep '^Makefile\.am:.*:=.*not portable' stderr
 grep README stderr && Exit 1
 $EGREP '(install|override)' stderr && Exit 1
diff --git a/tests/automake-cmdline.tap b/tests/automake-cmdline.tap
new file mode 100755
index 000000000..592c73b7c
--- /dev/null
+++ b/tests/automake-cmdline.tap
@@ -0,0 +1,60 @@
+#! /bin/sh
+# Copyright (C) 2004, 2006, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test Automake's command-line options.
+. ./defs || Exit 1
+
+# Usage: bad_cmdline DESCRIPTION REGEX-FOR-STDERR [ARGS-FOR-AUTOMAKE...]
+do_check ()
+{
+  test $# -ge 3 || fatal_ "do_check: invalid usage"
+  desc=$1; shift
+  regex=$1; shift
+  AUTOMAKE_fails -d "$desc (run)" -- "$@"
+  command_ok_ "$desc (stderr)" grep "$regex" stderr
+}
+
+do_check 'invalid long option' 'unrecognized option.*--voo' --voo
+
+# Older perl has a buggy Getopt::Long which makes this fail.
+if $PERL -e 'require 5.8.2;'; then
+  do_check "list of options terminated by '--'" \
+           'input file.*--voo' -- --voo
+else
+  skip_row_ 2 -r "older perl with buggy Getopt::Long"
+fi
+
+do_check "empty argument" \
+         'empty argument' ''
+
+do_check "missing argument for long option" \
+         'option.*-W.*requires an argument' -W
+
+do_check "missing argument for short option" \
+         'option.*--warnings.*requires an argument' --warnings
+
+do_check "'--help' as option argument" \
+         'unknown warning.*--help' --warnings --help
+
+do_check "'--help' as option argument" \
+         'unknown warning.*--help' --warnings --help
+
+do_check "ambiguous incomplete option" \
+         'unrecognized option.*--ver' --ver
+
+command_ok_ "unambiguous incomplete long option" $AUTOMAKE --vers
+
+:
diff --git a/tests/automake.test b/tests/automake.test
deleted file mode 100755
index db8c2b391..000000000
--- a/tests/automake.test
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2004, 2006, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# Test Automake's command-line options.
-. ./defs || Exit 1
-
-AUTOMAKE_fails --voo
-grep 'unrecognized option.*--voo' stderr
-# Older perl has a buggy Getopt::Long which makes this fail.
-if $PERL -e 'require 5.8.2;'; then
-  AUTOMAKE_fails -- --voo
-  grep 'input file.*--voo' stderr
-fi
-AUTOMAKE_fails ''
-grep 'empty argument' stderr
-AUTOMAKE_fails -W
-grep 'option.*-W.*requires an argument' stderr
-AUTOMAKE_fails --warnings
-grep 'option.*--warning.*requires an argument' stderr
-AUTOMAKE_fails --warnings --help
-grep 'unknown warning.*--help' stderr
-AUTOMAKE_fails --ver
-grep 'unrecognized option.*--ver' stderr
-$AUTOMAKE --vers
-
-:
diff --git a/tests/cond44.test b/tests/cond44.test
index 3d68a416e..ea5e87d03 100755
--- a/tests/cond44.test
+++ b/tests/cond44.test
@@ -32,7 +32,7 @@ endif
 EOF
 
 $ACLOCAL
-AUTOMAKE_run 0
+AUTOMAKE_run
 grep 'libdir was already defined' stderr && Exit 1
 grep '^libdir = ' Makefile.in && Exit 1
 Exit 0
diff --git a/tests/cond45.test b/tests/cond45.test
index ebaa2afd6..05d68dcf3 100755
--- a/tests/cond45.test
+++ b/tests/cond45.test
@@ -33,7 +33,7 @@ endif
 EOF
 
 $ACLOCAL
-AUTOMAKE_run 0
+AUTOMAKE_run
 grep 'foo was already defined' stderr && Exit 1
 grep '^foo =' Makefile.in && Exit 1
 Exit 0
diff --git a/tests/configure.test b/tests/configure.test
index 0e6290659..f9954afd6 100755
--- a/tests/configure.test
+++ b/tests/configure.test
@@ -47,7 +47,7 @@ AUTOMAKE_fails -Werror
 grep 'configure\.ac.*configure\.in.*both present' stderr
 grep 'proceeding.*configure\.ac' stderr
 
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
 grep 'configure\.ac.*configure\.in.*both present' stderr
 grep 'proceeding.*configure\.ac' stderr
 
diff --git a/tests/defs b/tests/defs
index da4fd7456..3ce92fd48 100644
--- a/tests/defs
+++ b/tests/defs
@@ -202,28 +202,78 @@ is_newest ()
   test -z "$is_newest_files"
 }
 
-# AUTOMAKE_run status [options...]
-# --------------------------------
-# Run Automake with OPTIONS, and fail if automake
-# does not exit with STATUS.
+# show_ [--prepend STRING] [FILES...]
+# ----------------------------
+# Show the content of each of the given files that is not empty,
+# prepending every one of its lines with the given STRING (defaulting
+# to " | ").  STRING cannot contain the "#" or "\" character.
+# This function is not just eye-candy and syntactic sugar: in tests
+# that use the TAP protocols, a line beginning with a "#" character
+# would be interpreted by the TAP driver as a diagnostic line, and
+# possibly displayed on console (with the risk of cluttering precious
+# screen space with noise).
+show_ ()
+{
+  set +x
+  p_=" | "
+  case $1 in -p|--prepend) p_=$2; shift 2;; esac
+  for f_ in ${1+"$@"}; do
+    test -s "$f_" || { echo "== $f_ is empty"; continue; }
+    echo "== $f_ =="
+    sed "s#^#$p_#" "$f_"
+  done
+  set -x
+}
+
+# AUTOMAKE_run [-e STATUS] [-d DESCRIPTION] [--] [AUTOMAKE-ARGS...]
+# -----------------------------------------------------------------
+# Run automake with AUTOMAKE-ARGS, and fail if it doesn't exit with
+# STATUS.  Should be polymorphic for TAP and "plain" tests.  The
+# DESCRIPTION, when provided, is used for console reporting, only if
+# the TAP protocol is in use in the current test script.
 AUTOMAKE_run ()
 {
-  expected_exitcode=$1
-  shift
-  exitcode=0
-  $AUTOMAKE ${1+"$@"} >stdout 2>stderr || exitcode=$?
+  am__desc=
+  am__exp_rc=0
+  while test $# -gt 0; do
+    case $1 in
+      -d) am__desc=$2; shift;;
+      -e) am__exp_rc=$2; shift;;
+      --) shift; break;;
+       # Don't fail on unknown option: assume they (and the rest of the
+       # command line) are to be passed verbatim to automake (so stop our
+       # own option parsing).
+       *) break;;
+    esac
+    shift
+  done
+  am__got_rc=0
+  $AUTOMAKE ${1+"$@"} >stdout 2>stderr || am__got_rc=$?
   cat stderr >&2
   cat stdout
-  test $exitcode = $expected_exitcode || Exit 1
+  if test $using_tap != yes; then
+    test $am__got_rc -eq $am__exp_rc || Exit 1
+    return
+  fi
+  if test -z "$am__desc"; then
+    if test $am__got_rc -eq $am__exp_rc; then
+      am__desc="automake exited $am__got_rc"
+    else
+      am__desc="automake exited $am__got_rc, expecting $am__exp_rc"
+    fi
+  fi
+  command_ok_ "$am__desc" test $am__got_rc -eq $am__exp_rc
 }
 
-# AUTOMAKE_fails [options...]
-# ---------------------------
-# Run Automake with OPTIONS, and fail if automake
-# does not exit with STATUS.
+# AUTOMAKE_fails [-d DESCRIPTION] [OPTIONS...]
+# --------------------------------------------
+# Run automake with OPTIONS, and fail if doesn't exit with status `1'.
+# Should be polymorphic for TAP and "plain" tests.  The DESCRIPTION,
+# when provided, is used for console reporting, only if the TAP
+# protocol is in use in the current test script.
 AUTOMAKE_fails ()
 {
-  AUTOMAKE_run 1 ${1+"$@"}
+  AUTOMAKE_run -e 1 ${1+"$@"}
 }
 
 # using_gmake
diff --git a/tests/getopt.test b/tests/getopt.test
index 9889e3dff..41aca0ca3 100755
--- a/tests/getopt.test
+++ b/tests/getopt.test
@@ -24,7 +24,7 @@ AUTOMAKE_fails -Wnonexistent
 grep ':.*nonexistent' stderr
 
 # ... but this should not.
-AUTOMAKE_run 0 -Wnonexistent --help
+AUTOMAKE_run -Wnonexistent --help
 grep ':.*nonexistent' stderr && Exit 1
 
 
@@ -33,7 +33,7 @@ AUTOMAKE_fails --nonexistent
 grep ':.*nonexistent' stderr
 
 # ... but this should not.
-AUTOMAKE_run 0 --nonexistent --help
+AUTOMAKE_run --nonexistent --help
 grep ':.*nonexistent' stderr && Exit 1
 
 :
diff --git a/tests/license2.test b/tests/license2.test
index 34bfdf80b..ca6a249d1 100755
--- a/tests/license2.test
+++ b/tests/license2.test
@@ -32,7 +32,7 @@ $ACLOCAL
 AUTOMAKE_fails
 grep 'COPYING' stderr
 
-AUTOMAKE_run 0 --add-missing
+AUTOMAKE_run --add-missing
 grep 'COPYING' stderr
 grep 'GNU General Public License' stderr
 grep 'Consider adding.*version control' stderr
diff --git a/tests/parallel-am.test b/tests/parallel-am.test
index 0182b44e1..453f7a52f 100755
--- a/tests/parallel-am.test
+++ b/tests/parallel-am.test
@@ -81,7 +81,7 @@ $ACLOCAL
 
 # Generate expected output using the non-threaded code.
 unset AUTOMAKE_JOBS || :
-AUTOMAKE_run 0 --add-missing
+AUTOMAKE_run --add-missing
 mv stderr expected
 Makefile_ins=`find . -name Makefile.in`
 for file in $Makefile_ins; do
@@ -93,7 +93,7 @@ export AUTOMAKE_JOBS
 
 for run in 1 2 3 4 5 6 7; do
   rm -f build-aux/* sub*/Makefile.in
-  AUTOMAKE_run 0 --add-missing
+  AUTOMAKE_run --add-missing
   diff stderr expected
   for file in $Makefile_ins; do
     diff $file $file.exp
diff --git a/tests/parallel-am3.test b/tests/parallel-am3.test
index 6db3ca7e3..732bd41ea 100755
--- a/tests/parallel-am3.test
+++ b/tests/parallel-am3.test
@@ -57,7 +57,7 @@ $ACLOCAL
 
 # Generate expected output using the non-threaded code.
 unset AUTOMAKE_JOBS || :
-AUTOMAKE_run 0 --add-missing
+AUTOMAKE_run --add-missing
 mv stderr expected
 mv Makefile.in Makefile.in.exp
 
@@ -66,7 +66,7 @@ export AUTOMAKE_JOBS
 
 for run in 1 2 3 4 5 6 7; do
   rm -f build-aux/* sub*/Makefile.in
-  AUTOMAKE_run 0 --add-missing
+  AUTOMAKE_run --add-missing
   diff stderr expected
   diff Makefile.in Makefile.in.exp
 done
diff --git a/tests/tags2.test b/tests/tags2.test
index aba350bdc..60e6f46ec 100755
--- a/tests/tags2.test
+++ b/tests/tags2.test
@@ -36,7 +36,7 @@ cat >>Makefile.am << 'END'
 bin_PROGRAMS = bar
 END
 
-AUTOMAKE_run 0
+AUTOMAKE_run
 grep 'define.*TAGS_DEPENDENCIES.*without' stderr && Exit 1
 
 :
diff --git a/tests/warnings-win-over-strictness.test b/tests/warnings-win-over-strictness.test
index 0db3176a4..83fced7a3 100755
--- a/tests/warnings-win-over-strictness.test
+++ b/tests/warnings-win-over-strictness.test
@@ -24,13 +24,13 @@ AUTOMAKE=$original_AUTOMAKE
 
 ok ()
 {
-  AUTOMAKE_run 0 $*
+  AUTOMAKE_run $*
   test ! -s stderr
 }
 
 ko ()
 {
-  AUTOMAKE_run 0 $*
+  AUTOMAKE_run $*
   grep '^Makefile\.am:.*:=.*not portable' stderr
   test `wc -l <stderr` -eq 1
 }
diff --git a/tests/werror3.test b/tests/werror3.test
index 98cc8f0da..33ad6f726 100755
--- a/tests/werror3.test
+++ b/tests/werror3.test
@@ -37,11 +37,11 @@ $ACLOCAL
 
 # The issue would not manifest with threaded execution.
 unset AUTOMAKE_JOBS || :
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
 grep 'VAR multiply defined' stderr
 
 AUTOMAKE_JOBS=2
 export AUTOMAKE_JOBS
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
 grep 'VAR multiply defined' stderr
 :
diff --git a/tests/werror4.test b/tests/werror4.test
index 2ffaa4eeb..df41bdc51 100755
--- a/tests/werror4.test
+++ b/tests/werror4.test
@@ -58,7 +58,7 @@ grep 'VAR multiply defined' stderr
 grep 'SUB multiply defined' stderr
 test `grep -c 'warnings are treated as errors' stderr` -eq 1
 
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
 grep 'VAR multiply defined' stderr
 grep 'SUB multiply defined' stderr
 grep 'warnings are treated as errors' stderr && Exit 1