summaryrefslogtreecommitdiff
path: root/tests/testsuite-summary-count-many.test
blob: 58157ad9b5312144c605da064b714a837c2ea313 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#! /bin/sh
# Copyright (C) 2011-2012 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Check test counts in the testsuite summary, with test drivers allowing
# multiple test results per test script, and for a huge number of tests.
# Incidentally, this test also checks that the testsuite summary doesn't
# give any bug-report address if it's not defined.

am_parallel_tests=yes
. ./defs || Exit 1

for s in trivial-test-driver extract-testsuite-summary.pl; do
  cp "$am_testauxdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
done

br='============================================================================'

header="\
${br}
Testsuite summary for $me 1.0
${br}"

footer="\
${br}
See ./test-suite.log
${br}"

echo AC_OUTPUT >> configure.ac

cat > Makefile.am << 'END'
TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
TESTS = all.test
# Without this, the test driver will be horrendously slow.
END

cat > all.test <<'END'
#!/bin/sh
cat results.txt || { echo ERROR: weird; exit 99; }
END
chmod a+x all.test

$PERL -w -e '
  use warnings FATAL => "all";
  use strict;

  # FIXME: we would like this to be 1000 or even 10000, but the current
  # implementation is too slow to handle that :-(
  my $base = 5;
  my %count = (
    TOTAL => $base * 1000,
    PASS  => $base * 700,
    SKIP  => $base * 200,
    XFAIL => $base * 80,
    FAIL  => $base * 10,
    XPASS => $base * 7,
    ERROR => $base * 3,
  );
  my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/;

  open (RES, ">results.txt") or die "opening results.txt: $!\n";
  open (CNT, ">count.txt") or die "opening count.txt: $!\n";

  printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL};
  for my $res (@results)
    {
      my $uc_res = uc $res;
      print STDERR "Generating list of $res ...\n";
      for (1..$count{$res})
        {
          print RES "$uc_res: $_\n";
        }
      printf CNT "# %-6s %d\n", $res . ":", $count{$res};
    }
'

(echo "$header" && cat count.txt && echo "$footer") > summary.exp

$ACLOCAL
$AUTOMAKE -a
$AUTOCONF

./configure

($MAKE check || : > make.fail) | tee stdout
test -f make.fail

$PERL extract-testsuite-summary.pl stdout > summary.got
cat summary.exp
cat summary.got
diff summary.exp summary.got || Exit 1

: