path: root/t/ax/tap-summary-aux.sh

#! /bin/sh
# Copyright (C) 2011-2012 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Auxiliary script for tests on TAP support: checking testsuite summary.

am_parallel_tests=yes
. ./defs || Exit 1

br='============================================================================'

case $use_colors in
  yes|no) ;;
  *) fatal_ "invalid \$use_colors value '$use_colors'"
esac

fetch_tap_driver

cat > configure.ac <<END
AC_INIT([GNU AutoTAP], [5.12], [bug-automake@gnu.org])
AM_INIT_AUTOMAKE([parallel-tests])
AC_CONFIG_FILES([Makefile])
AC_OUTPUT
END

cat > Makefile.am << 'END'
TEST_LOG_DRIVER = $(srcdir)/tap-driver
TEST_LOG_COMPILER = cat
TESTS = all.test
END

# The following shell variables are influential for this function:
#  - expect_failure
#  - use_colors
do_check ()
{
  case $#,$1 in
    1,--pass) expect_failure=no;;
    1,--fail) expect_failure=yes;;
           *) fatal_ "invalid usage of 'do_check'";;
  esac
  shift
  cat > summary.exp
  cat all.test
  st=0
  if test $use_colors = yes; then
    make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
  else
    make_cmd=$MAKE
  fi
  $make_cmd check > stdout || st=$?
  cat stdout
  if test $expect_failure = yes; then
    test $st -gt 0 || Exit 1
  else
    test $st -eq 0 || Exit 1
  fi
  $PERL "$am_testauxdir"/extract-testsuite-summary.pl stdout >summary.got \
    || fatal_ "cannot extract testsuite summary"
  cat summary.exp
  cat summary.got
  if test $use_colors = yes; then
    # Use cmp, not diff, because the files might contain binary data.
    compare=cmp
  else
    compare=diff
  fi
  $compare summary.exp summary.got || Exit 1
}

if test $use_colors = yes; then
  red=''
  grn=''
  lgn=''
  blu=''
  mgn=''
  brg=''
  std=''
  echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
else
  red= grn= lgn= blu= mgn= brg= std=
fi

  success_header="\
${grn}${br}${std}
${grn}Testsuite summary for GNU AutoTAP 5.12${std}
${grn}${br}${std}"

  success_footer=${grn}${br}${std}

  failure_header="\
${red}${br}${std}
${red}Testsuite summary for GNU AutoTAP 5.12${std}
${red}${br}${std}"

  failure_footer="\
${red}${br}${std}
${red}See ./test-suite.log${std}
${red}Please report to bug-automake@gnu.org${std}
${red}${br}${std}"

$ACLOCAL
$AUTOCONF
$AUTOMAKE --add-missing

./configure

# 1 pass.
{ echo 1..1 && echo ok; } > all.test
do_check --pass <<END
$success_header
${brg}# TOTAL: 1${std}
${grn}# PASS:  1${std}
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

# 1 skip.
{ echo 1..1 && echo 'ok # SKIP'; } > all.test
do_check --pass <<END
$success_header
${brg}# TOTAL: 1${std}
# PASS:  0
${blu}# SKIP:  1${std}
# XFAIL: 0
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

# 1 xfail.
{ echo 1..1 && echo 'not ok # TODO'; } > all.test
do_check --pass <<END
$success_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
${lgn}# XFAIL: 1${std}
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

# 1 fail.
{ echo 1..1 && echo not ok; } > all.test
do_check --fail <<END
$failure_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
# XFAIL: 0
${red}# FAIL:  1${std}
# XPASS: 0
# ERROR: 0
$failure_footer
END

# 1 xpass.
{ echo 1..1 && echo 'ok # TODO'; } > all.test
do_check --fail <<END
$failure_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
# XFAIL: 0
# FAIL:  0
${red}# XPASS: 1${std}
# ERROR: 0
$failure_footer
END

# 1 hard error.
{ echo 1..1 && echo 'Bail out!'; } > all.test
do_check --fail <<END
$failure_header
${brg}# TOTAL: 1${std}
# PASS:  0
# SKIP:  0
# XFAIL: 0
# FAIL:  0
# XPASS: 0
${mgn}# ERROR: 1${std}
$failure_footer
END

# 3 non-failing results.
cat > all.test <<END
1..3
ok
not ok # TODO
ok # SKIP
END
do_check --pass <<END
$success_header
${brg}# TOTAL: 3${std}
${grn}# PASS:  1${std}
${blu}# SKIP:  1${std}
${lgn}# XFAIL: 1${std}
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

# 1 pass, 1 skip, 1 fail.
cat > all.test <<END
1..3
ok
ok # SKIP
not ok
END
do_check --fail <<END
$failure_header
${brg}# TOTAL: 3${std}
${grn}# PASS:  1${std}
${blu}# SKIP:  1${std}
# XFAIL: 0
${red}# FAIL:  1${std}
# XPASS: 0
# ERROR: 0
$failure_footer
END

# 1 pass, 1 xfail, 1 xpass.
cat > all.test <<END
1..3
ok
ok # TODO
not ok # TODO
END
do_check --fail <<END
$failure_header
${brg}# TOTAL: 3${std}
${grn}# PASS:  1${std}
# SKIP:  0
${lgn}# XFAIL: 1${std}
# FAIL:  0
${red}# XPASS: 1${std}
# ERROR: 0
$failure_footer
END

# 1 skip, 1 xfail, 1 error.
cat > all.test <<END
1..3
ok # SKIP
not ok # TODO
Bail out!
END
do_check --fail <<END
$failure_header
${brg}# TOTAL: 3${std}
# PASS:  0
${blu}# SKIP:  1${std}
${lgn}# XFAIL: 1${std}
# FAIL:  0
# XPASS: 0
${mgn}# ERROR: 1${std}
$failure_footer
END

# 1 of each kind
cat > all.test <<END
1..6
ok
not ok
ok # TODO
not ok # TODO
ok # SKIP
Bail out!
END
do_check --fail <<END
$failure_header
${brg}# TOTAL: 6${std}
${grn}# PASS:  1${std}
${blu}# SKIP:  1${std}
${lgn}# XFAIL: 1${std}
${red}# FAIL:  1${std}
${red}# XPASS: 1${std}
${mgn}# ERROR: 1${std}
$failure_footer
END

# Prepare some common data for later.
for i in 0 1 2 3 4 5 6 7 8 9; do
  for j in 0 1 2 3 4 5 6 7 8 9; do
    echo "ok"
    echo "not ok # TODO"
    echo "ok # SKIP"
  done
done > tap

# Lots of non-failures (300 per kind).
(cat tap && cat tap && cat tap) > all.test
test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
echo 1..900 >> all.test # Test plan.
do_check --pass <<END
$success_header
${brg}# TOTAL: 900${std}
${grn}# PASS:  300${std}
${blu}# SKIP:  300${std}
${lgn}# XFAIL: 300${std}
# FAIL:  0
# XPASS: 0
# ERROR: 0
$success_footer
END

# 1 failure and lots of non-failures means failure.
(cat tap && echo "not ok" && cat tap) > all.test
test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
echo 1..601 >> all.test # Test plan.
do_check --fail <<END
$failure_header
${brg}# TOTAL: 601${std}
${grn}# PASS:  200${std}
${blu}# SKIP:  200${std}
${lgn}# XFAIL: 200${std}
${red}# FAIL:  1${std}
# XPASS: 0
# ERROR: 0
$failure_footer
END

# 1 error and lots of non-failures means failure.
(cat tap && sed 30q tap && echo 'Bail out!') > all.test
test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
echo 1..331 >> all.test # Test plan.
do_check --fail <<END
$failure_header
${brg}# TOTAL: 331${std}
${grn}# PASS:  110${std}
${blu}# SKIP:  110${std}
${lgn}# XFAIL: 110${std}
# FAIL:  0
# XPASS: 0
${mgn}# ERROR: 1${std}
$failure_footer
END

: