## automake - create Makefile.in from Makefile.am
## Copyright (C) 2001-2014 Free Software Foundation, Inc.
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2, or (at your option)
## any later version.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with this program. If not, see .
## New parallel test driver.
##
## The first version of the code here was adapted from check.mk, which was
## originally written at EPITA/LRDE, further developed at Gostai, then made
## its way from GNU coreutils to end up, largely rewritten, in Automake.
## The current version is an heavy rewrite of that, to allow for support
## of more test metadata, and the use of custom test drivers and protocols
## (among them, TAP).
## Used by (at least) 'check-typos.am'.
am.conf.using-parallel-tests := yes
am.test-suite.is-xfail = \
$(if $(filter-out $(am.test-suite.xfail-test-bases), \
$(patsubst $(srcdir)/%,%,$(1))),no,yes)
am.test-suite.runtest = \
$(am.test-suite.tty-colors); \
if test '$(TEST_SUITE_LOG)' = '$*.log'; then \
echo "fatal: $*.log: depends on itself (check TESTS content)" >&2; \
exit 1; \
fi; \
srcdir=$(srcdir); export srcdir; \
## Creates the directory for the log file if needed. Avoid extra forks.
test x$(@D) = x. || test -d $(@D) || $(MKDIR_P) $(@D) || exit $$?; \
## We need to invoke the test in way that won't cause a PATH search.
## Quotes around '$<' are required to avoid extra errors when a circular
## dependency is detected (e.g., because $(TEST_SUITE_LOG) is in
## $(am.test-suite.test-logs)), because in that case '$<' expands to empty and an
## unquote usage of it could cause syntax errors in the shell.
case '$<' in */*) tst='$<';; *) tst=./'$<';; esac; \
## Executes the developer-defined and user-defined test
## setups (if any), in that order.
$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
$($(1)LOG_DRIVER) \
--test-name '$(patsubst $(srcdir)/%,%,$<)' \
--log-file $*.log \
--trs-file $*.trs \
--color-tests "$$am__color_tests" \
--enable-hard-errors $(if $(DISABLE_HARD_ERRORS),no,yes) \
--expect-failure $(call am.test-suite.is-xfail,$*) \
$(AM_$(1)LOG_DRIVER_FLAGS) \
$($(1)LOG_DRIVER_FLAGS) \
-- \
$($(1)LOG_COMPILER) \
$(AM_$(1)LOG_FLAGS) \
$($(1)LOG_FLAGS) \
"$$tst" \
$(AM_TESTS_FD_REDIRECT)
define am.test-suite.handle-suffix.helper
ifeq ($$(call am.vars.is-undef,$(2)LOG_DRIVER),yes)
$(2)LOG_DRIVER = $(SHELL) $(am.conf.aux-dir)/test-driver
endif
%.log %.trs: %$1 $$($(2)LOG_DEPENDENCIES)
@$$(call am.test-suite.runtest,$2)
ifdef EXEEXT
%.log %.trs: %$1$(EXEEXT) $$($(2)LOG_DEPENDENCIES)
@$$(call am.test-suite.runtest,$2)
endif
endef
define am.test-suite.handle-suffix
$(call $0.helper,$1,$(if $1,$(call am.util.toupper,$(patsubst .%,%_,$1))))
endef
ifeq ($(call am.vars.is-undef,TEST_EXTENSIONS),yes)
TEST_EXTENSIONS := .test
endif
$(foreach e,$(filter-out .%,$(TEST_EXTENSIONS)),\
$(call am.error,invalid test extension: '$e'))
$(foreach e,$(TEST_EXTENSIONS), \
$(eval $(call am.test-suite.handle-suffix,$e)))
# It is *imperative* that the "empty" suffix goes last. Otherwise, a
# declaration like "TESTS = all.test" would cause GNU make to mistakenly
# try to build the 'all.log' and 'all.trs' files from a non-existent
# 'all' program (because the Makefile contains an explicit 'all' target,
# albeit .PHONY), rather than from the 'all.test' script, thus causing
# all sort of mishaps and confusion.
$(eval $(call am.test-suite.handle-suffix))
# The names of the given tests scripts with any possible registered
# test extension removed, as well as any leading '$(srcdir)' component
# (if any) stripped.
# The stripping of $(srcdir) is required to support explicit use of
# $(srcdir) in TESTS entries. That might actually be very useful in
# practice, for example in usages like this:
# TESTS = $(wildcard $(srcdir)/t[0-9][0-9]*.sh)
# where removing the $(srcdir) from the $(wildcard) invocation would
# cause the idiom to break in VPATH builds.
define am.test-suite.get-test-bases
$(patsubst $(srcdir)/%,%,$(strip \
$(call am.util.strip-suffixes, $(TEST_EXTENSIONS), \
$(if $(EXEEXT),$(patsubst %$(EXEEXT),%,$1),$1))))
endef
am.test-suite.rx.recheck = ^[ ]*:recheck:[ ]*
am.test-suite.rx.global-result = ^[ ]*:global-test-result:[ ]*
am.test-suite.rx.result = ^[ ]*:test-result:[ ]*
am.test-suite.rx.copy-in-global-log = ^[ ]*:copy-in-global-log:[ ]*
# Some awk code fragments used by one another and eventually by the
# 'check' and 'recheck' recipes.
# Note that in those scripts we are careful to close all the '.trs' and
# '.log' files once we are done with them. This is done to avoid leaking
# open file descriptors, which could cause serious problems when there
# are many tests and thus lots of '.log' and '.trs' files to open (yes,
# there would be problems even on Linux).
am.test-suite.awk-functions = \
function error(msg) \
{ \
print msg | "cat >&2"; \
exit_status = 1; \
} \
function input_error(file) \
{ \
error("awk" ": cannot read \"" file "\""); \
## Never leak file descriptors, not even on errors.
close ($$0 ".trs"); close ($$0 ".log"); \
}
# Loop on the lines in the current '.trs' or '.log' file,
# punting on I/O errors.
am.test-suite.awk-io-loop/BEGIN = \
while ((rc = (getline line < ($$0 ".$1"))) != 0) { \
if (rc < 0) { input_error($$0 ".$1"); next; }
am.test-suite.awk-io-loop/END = \
}; close ($$0 ".$1");
# A command that, given a newline-separated list of test names on the
# standard input, print the name of the tests that are to be re-run
# upon "make recheck".
am.test-suite.list-tests-to-recheck = $(AWK) '{ \
## By default, we assume the test is to be re-run.
recheck = 1; \
while ((rc = (getline line < ($$0 ".trs"))) != 0) \
{ \
if (rc < 0) \
{ \
##
## If we have encountered an I/O error here, there are three possibilities:
##
## [1] The '.log' file exists, but the '.trs' does not; in this case,
## we "gracefully" recover by assuming the corresponding test is
## to be re-run (which will re-create the missing '.trs' file).
##
## [2] Both the '.log' and '.trs' files are missing; this means that
## the corresponding test has not been run, and is thus *not* to
## be re-run.
##
## [3] We have encountered some corner-case problem (e.g., a '.log' or
## '.trs' files somehow made unreadable, or issues with a bad NFS
## connection, or whatever); we do not handle such corner cases.
##
if ((getline line2 < ($$0 ".log")) < 0) \
recheck = 0; \
break; \
} \
else if (line ~ /$(am.test-suite.rx.recheck)[nN][Oo]/) \
## A directive explicitly specifying the test is *not* to be re-run.
{ \
recheck = 0; \
break; \
} \
else if (line ~ /$(am.test-suite.rx.recheck)[yY][eE][sS]/) \
{ \
## A directive explicitly specifying the test *is* to be re-run.
break; \
} \
## else continue with the next iteration.
}; \
if (recheck) \
print $$0; \
## Never leak file descriptors.
close ($$0 ".trs"); close ($$0 ".log"); \
}'
# A command that, given a newline-separated list of test names on the
# standard input, output a shell code snippet setting variables that
# count occurrences of each test result (PASS, FAIL, etc) declared in
# the '.trs' files of that given tests. For example, the count of
# PASSes will be saved in the '$am_PASS' variable, the count of SKIPs
# in the '$am_SKIP' variable, and so on.
am.test-suite.count-results = $(AWK) ' \
$(am.test-suite.awk-functions) \
BEGIN { exit_status = 0; } \
{ \
$(call am.test-suite.awk-io-loop/BEGIN,trs) \
if (line ~ /$(am.test-suite.rx.result)/) \
{ \
sub("$(am.test-suite.rx.result)", "", line); \
sub("[: ].*$$", "", line); \
counts[line]++;\
} \
$(call am.test-suite.awk-io-loop/END,trs) \
} \
END { \
if (exit_status != 0) \
error("fatal: making $@: I/O error reading test results"); \
else \
{ \
global_count = 0; \
for (k in counts) \
{ \
print "am_" k "=" counts[k]; \
global_count += counts[k]; \
} \
} \
print "am_ALL=" global_count; \
exit(exit_status); \
}'
# A command that, given a newline-separated list of test names on the
# standard input, create the global log from their .trs and .log files.
am.test-suite.create-global-log = $(AWK) ' \
$(am.test-suite.awk-functions) \
function rst_section(header) \
{ \
print header; \
len = length(header); \
for (i = 1; i <= len; i = i + 1) \
printf "="; \
printf "\n\n"; \
} \
BEGIN { exit_status = 0; } \
{ \
## By default, we assume the test log is to be copied in the global log,
## and that its result is simply "RUN" (i.e., we still do not know what
## it outcome was, but we know that at least it has run).
copy_in_global_log = 1; \
global_test_result = "RUN"; \
$(call am.test-suite.awk-io-loop/BEGIN,trs) \
if (line ~ /$(am.test-suite.rx.global-result)/) \
{ \
sub("$(am.test-suite.rx.global-result)", "", line); \
sub("[ ]*$$", "", line); \
global_test_result = line; \
} \
else if (line ~ /$(am.test-suite.rx.copy-in-global-log)[nN][oO]/) \
copy_in_global_log = 0; \
$(call am.test-suite.awk-io-loop/END,trs) \
if (copy_in_global_log) \
{ \
rst_section(global_test_result ": " $$0); \
$(call am.test-suite.awk-io-loop/BEGIN,log) \
print line; \
$(call am.test-suite.awk-io-loop/END,log) \
printf "\n"; \
}; \
} \
END { \
if (exit_status != 0) \
error("fatal: making $@: I/O error reading test results"); \
exit(exit_status); \
}'
# Restructured Text title.
am.test-suite.rst-title = { sed 's/.*/ & /;h;s/./=/g;p;x;s/ *$$//;p;g' && echo; }
# These support runtime overriding of $(TESTS) and $(XFAIL_TESTS).
# The first one must be left overridable (hence the definition with '?=',
# because the 'recheck' target need to override it (and in a tricky way).
am.test-suite.test-bases ?= \
$(call am.memoize,am.test-suite.test-bases,$(call am.test-suite.get-test-bases,$(TESTS)))
am.test-suite.xfail-test-bases = \
$(call am.memoize,am.test-suite.xfail-test-bases,$(call am.test-suite.get-test-bases,$(XFAIL_TESTS)))
# The $(strip) is to work around the GNU make 3.80 bug where trailing
# whitespace in "TESTS = foo.test $(empty)" causes $(TESTS_LOGS) to
# erroneously expand to "foo.log .log".
am.test-suite.test-results = \
$(call am.memoize,am.test-suite.test-results,$(addsuffix .trs,$(strip $(am.test-suite.test-bases))))
am.test-suite.test-logs = \
$(call am.memoize,am.test-suite.test-logs,$(addsuffix .log,$(strip $(am.test-suite.test-bases))))
am.clean.mostly.f += $(am.test-suite.test-results) $(am.test-suite.test-logs)
# $(TEST_LOGS) is a published interface.
TEST_LOGS = $(am.test-suite.test-logs)
am.test-suite.workdir = $(am.dir)/test-harness
am.test-suite.append-to-list-of-bases = \
@lst='$1'; for x in $$lst; do echo $$x; done \
>> $(am.test-suite.workdir)/bases$(am.chars.newline)
define am.setup-test-harness-workdir
@rm -rf $(am.test-suite.workdir)
@$(MKDIR_P) $(am.test-suite.workdir)
@touch $(am.test-suite.workdir)/bases
$(call am.xargs-map,am.test-suite.append-to-list-of-bases, \
$(am.test-suite.test-bases))
@workdir='$(am.test-suite.workdir)' \
&& sed 's/$$/.log/' $$workdir/bases > $$workdir/logs \
&& sed 's/$$/.trs/' $$workdir/bases > $$workdir/trs
endef
ifeq ($(call am.vars.is-undef,TEST_SUITE_LOG),yes)
TEST_SUITE_LOG = test-suite.log
endif
$(TEST_SUITE_LOG): $(am.test-suite.test-logs) $(am.test-suite.test-results)
$(am.setup-test-harness-workdir)
@set +e; $(am.test-suite.tty-colors); \
fatal () { echo "fatal: making $@: $$*" >&2; exit 1; }; \
workdir='$(am.test-suite.workdir)'; \
## Prepare data for the test suite summary. These do not take into account
## unreadable test results, but they'll be appropriately updated later if
## needed.
am_PASS=0 am_FAIL=0 am_SKIP=0 am_XPASS=0 am_XFAIL=0 am_ERROR=0; \
count_test_results_command=`\
$(am.test-suite.count-results) <$$workdir/bases` \
&& eval "$$count_test_results_command" \
|| fatal "unknown error reading test results"; \
## Whether the testsuite was successful or not.
if test `expr $$am_FAIL + $$am_XPASS + $$am_ERROR` -eq 0; then \
success=true; \
else \
success=false; \
fi; \
## Make $br a line of exactly 76 '=' characters, that will be used to
## enclose the testsuite summary report when displayed on the console.
br='==================='; br=$$br$$br$$br$$br; \
## When writing the test summary to the console, we want to color a line
## reporting the count of some result *only* if at least one test
## experienced such a result. This function is handy in this regard.
display_result_count () \
{ \
if test x"$$1" = x"--maybe-color"; then \
maybe_colorize=yes; \
elif test x"$$1" = x"--no-color"; then \
maybe_colorize=no; \
else \
echo "$@: invalid 'display_result_count' usage" >&2; \
exit 4; \
fi; \
shift; \
desc=$$1 count=$$2; \
if test $$maybe_colorize = yes && test $$count -gt 0; then \
color_start=$$3 color_end=$$std; \
else \
color_start= color_end=; \
fi; \
echo "$${color_start}# $$desc $$count$${color_end}"; \
}; \
## A shell function that creates the testsuite summary. We need it
## because we have to create *two* summaries, one for test-suite.log,
## and a possibly-colorized one for console output.
create_testsuite_report () \
{ \
opts=$$*; \
display_result_count $$opts "TOTAL:" $$am_ALL "$$brg"; \
display_result_count $$opts "PASS: " $$am_PASS "$$grn"; \
display_result_count $$opts "SKIP: " $$am_SKIP "$$blu"; \
display_result_count $$opts "XFAIL:" $$am_XFAIL "$$lgn"; \
display_result_count $$opts "FAIL: " $$am_FAIL "$$red"; \
display_result_count $$opts "XPASS:" $$am_XPASS "$$red"; \
display_result_count $$opts "ERROR:" $$am_ERROR "$$mgn"; \
}; \
## Write "global" testsuite log.
if { \
st=0; \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am.test-suite.rst-title); \
create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
$(am.test-suite.create-global-log) <$$workdir/bases; \
} >$(TEST_SUITE_LOG).tmp; then \
mv -f $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
else \
## The awk program in $(am.test-suite.create-global-log) should have already
## emitted a proper error message about I/O error, no need to repeat it.
rm -f $(TEST_SUITE_LOG).tmp; exit 1; \
fi; \
## Emit the test summary on the console.
if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
fi; \
## Multi line coloring is problematic with "less -R", so we really need
## to color each line individually.
echo "$${col}$$br$${std}"; \
echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
echo "$${col}$$br$${std}"; \
## This is expected to go to the console, so it might have to be colorized.
create_testsuite_report --maybe-color; \
echo "$$col$$br$$std"; \
if $$success; then :; else \
echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
if test -n "$(PACKAGE_BUGREPORT)"; then \
echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
fi; \
echo "$$col$$br$$std"; \
fi; \
$$success || exit 1
am.clean.mostly.f += $(TEST_SUITE_LOG)
## ------------------------------------------ ##
## Running all tests, or rechecking failures. ##
## ------------------------------------------ ##
check-TESTS:
ifneq ($(AM_LAZY_CHECK),yes)
@$(call am.clean-cmd.f, \
$(am.test-suite.test-results) $(am.test-suite.test-logs))
endif
## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
## in any case even in lazy mode: otherwise, if no test needs rerunning,
## or a prior run plus reruns all happen within the same timestamp (can
## happen with a prior "make TESTS="), then we get no log output.
## OTOH, this means that, in the rule for '$(TEST_SUITE_LOG)', we
## cannot use '$?' to compute the set of lazily rerun tests, lest
## we rely on .PHONY to work portably.
@rm -f $(TEST_SUITE_LOG)
$(MAKE) $(TEST_SUITE_LOG)
.PHONY: check-TESTS
# Recheck must depend on $(check_SCRIPTS), $(check_PROGRAMS), etc.
# It must also depend on the 'all' target. See automake bug#11252.
recheck: all $(am.test-suite.deps)
+$(am.setup-test-harness-workdir)
## See comments above in the check-TESTS recipe for why remove
## $(TEST_SUITE_LOG) here.
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@bases=`$(am.test-suite.list-tests-to-recheck) \
<$(am.test-suite.workdir)/bases` || exit 1; \
## Remove newlines and normalize whitespace.
bases=`echo $$bases`; \
## Re-run the relevant tests, without hitting command-line length limits.
echo am.test-suite.test-bases="$$bases" | \
$(MAKE) -f- -f$(firstword $(MAKEFILE_LIST)) \
$(TEST_SUITE_LOG) .am/doing-recheck=yes
.PHONY: recheck
# One tricky requirement of the "recheck" target is that, in case (say)
# the test is a compiled program whose compilation fails, we must ensure
# that any '.log' and '.trs' file referring to such test are preserved,
# so that future "make recheck" invocations will still try to re-compile
# and re-run it (automake bug#11791). This indirection is aimed at
# satisfying such a requirement.
ifeq ($(.am/doing-recheck),yes)
$(am.test-suite.test-logs) $(am.test-suite.test-results): .am/nil
endif
AM_RECURSIVE_TARGETS += check recheck