1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
#! /bin/sh
# Copyright (C) 2009-2014 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Basic checks on parallel-tests support:
# - console output
# - log files, and what goes in 'test-suite.log'
# - make clean
# - dependencies between tests
# - TESTS redefinition at runtime (with and without test suffixes)
# - AM_LAZY_CHECK
. test-init.sh
cat >> configure.ac << 'END'
AC_OUTPUT
END
cat > Makefile.am << 'END'
TESTS = foo.test bar.test baz.test
XFAIL_TESTS = bar.test
foo.log: bar.log
bar.log: baz.log
END
cat > foo.test <<'END'
#! /bin/sh
echo "this is $0"
exit 0
END
cat > bar.test <<'END'
#! /bin/sh
echo "this is $0"
exit 99
END
cat > baz.test <<'END'
#! /bin/sh
echo "this is $0"
exit 1
END
chmod a+x foo.test bar.test baz.test
$ACLOCAL
$AUTOCONF
$AUTOMAKE -a
./configure
run_make -O -e FAIL check
count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
test -f test-suite.log
cat test-suite.log
test $(grep -c '^FAIL:' test-suite.log) -eq 1
test $(grep -c '^ERROR:' test-suite.log) -eq 1
$EGREP '^(X?PASS|XFAIL|SKIP)' test-suite.log && exit 1
test -f baz.log
test -f bar.log
test -f foo.log
$MAKE clean
test ! -e baz.log
test ! -e bar.log
test ! -e foo.log
test ! -e test-suite.log
# Check dependencies: baz.test needs to run before bar.test,
# but foo.test is not needed.
# Note that this usage has a problem: the summary will only
# take bar.log into account, because the $(TEST_SUITE_LOG) rule
# does not "see" baz.log. Hmm.
run_make -O -e FAIL TESTS='bar.test' check
grep '^FAIL: baz\.test$' stdout
grep '^ERROR: bar\.test$' stdout
test -f baz.log
test -f bar.log
test ! -e foo.log
test -f test-suite.log
# Upon a lazy rerun, foo.test should be run, but the others shouldn't.
# Note that the lazy rerun still exits with a failure, due to the previous
# test failures.
# Note that the previous test and this one taken together expose the timing
# issue that requires the check-TESTS rule to always remove TEST_SUITE_LOG
# before running the tests lazily.
run_make -O -e FAIL check AM_LAZY_CHECK=yes
test -f foo.log
grep '^PASS: foo\.test$' stdout
grep bar.test stdout && exit 1
grep baz.test stdout && exit 1
grep '^# PASS: *1$' stdout
grep '^# FAIL: *1$' stdout
grep '^# ERROR: *1$' stdout
# Now, explicitly retry with all test logs already updated, and ensure
# that the summary is still displayed.
run_make -O -e FAIL check AM_LAZY_CHECK=yes
grep foo.test stdout && exit 1
grep bar.test stdout && exit 1
grep baz.test stdout && exit 1
grep '^# PASS: *1$' stdout
grep '^# FAIL: *1$' stdout
grep '^# ERROR: *1$' stdout
$MAKE clean
run_make -O -e FAIL check TESTS=baz
grep foo.test stdout && exit 1
grep bar.test stdout && exit 1
grep baz.test stdout
$MAKE clean
run_make -O -e FAIL TESTS=baz.test check
grep foo.test stdout && exit 1
grep bar.test stdout && exit 1
grep baz.test stdout
:
|
