summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.travis.yml2
-rw-r--r--Makefile31
-rw-r--r--testsuite/Makefile5
-rw-r--r--testsuite/driver/testglobals.py4
-rw-r--r--testsuite/driver/testlib.py9
-rw-r--r--testsuite/mk/test.mk20
-rw-r--r--testsuite/tests/concurrent/should_run/all.T6
-rw-r--r--testsuite/tests/gadt/all.T2
-rwxr-xr-xtestsuite/tests/typecheck/should_run/all.T5
-rwxr-xr-xvalidate10
10 files changed, 64 insertions, 30 deletions
diff --git a/.travis.yml b/.travis.yml
index 660d38305a..3a9d337344 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -53,4 +53,6 @@ script:
# * But use VERBOSE=2 (the default, but not when using --quiet) otherwise
# the testsuite might not print output for over 10 minutes (more likely so
# when DEBUG_STAGE2=NO), causing Travis to again kill our job.
+ # * Use --fast, to stay within the time limits set by Travis.
+ # See Note [validate and testsuite speed] in toplevel Makefile.
- THREADS=3 SKIP_PERF_TESTS=YES VERBOSE=2 ./validate --fast --quiet
diff --git a/Makefile b/Makefile
index b1b1742eb3..e6a1bc5b02 100644
--- a/Makefile
+++ b/Makefile
@@ -83,6 +83,7 @@ REALGOALS=$(filter-out \
help \
test \
fulltest \
+ slowtest \
fasttest \
,$(MAKECMDGOALS))
@@ -159,10 +160,36 @@ endif
endif
+# Note [validate and testsuite speed]
+#
+# There are 3 different validate and testsuite speed settings:
+# fast, normal and slow.
+#
+# how how used
+# cd tests config. many many by
+# validate && make speed= tests ways whom
+# =============================================================================
+# --fast fast 2 some 1 Travis (to stay within time limit)
+# --normal test 1 all 1 Phabricator (slow takes too long?)
+# --slow slow 0 all all Nightly (slow is ok)
+#
+# accept 1 all 1
+#
+# `make accept` should run all tests exactly once. There is no point in
+# accepting a test for multiple ways, since it should produce the same output
+# for all ways.
+#
+# To make sure all .stderr and .stdout files in the testsuite are never
+# out-of-date, it is useful if Phabricator, via a normal `./validate` and `make
+# test`, runs each test at least once.
.PHONY: fasttest
fasttest:
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast
-.PHONY: fulltest test
-fulltest test:
+.PHONY: test
+test:
$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt
+
+.PHONY: slowtest fulltest
+slowtest fulltest:
+ $(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt slow
diff --git a/testsuite/Makefile b/testsuite/Makefile
index 6ad34393fc..661769b891 100644
--- a/testsuite/Makefile
+++ b/testsuite/Makefile
@@ -11,7 +11,7 @@ ifneq "$(MAKECMDGOALS)" "maintainer-clean"
include $(TOP)/mk/boilerplate.mk
-.PHONY: all boot test verbose accept fast list_broken
+.PHONY: all boot test verbose accept fast slow list_broken
boot:
$(MAKE) -C $(TOP)/timeout all
@@ -31,6 +31,9 @@ accept:
fast:
$(MAKE) -C $(TOP)/tests fast
+slow:
+ $(MAKE) -C $(TOP)/tests slow
+
list_broken:
$(MAKE) -C $(TOP)/tests list_broken
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 4b6754fa40..95168f3c5e 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -54,8 +54,8 @@ class TestConfig:
# Verbosity level
self.verbose = 3
- # run the "fast" version of the test suite
- self.fast = 0
+ # See Note [validate and testsuite speed] in toplevel Makefile.
+ self.speed = 1
self.list_broken = False
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index c6a3bc2461..68a9208a4b 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -333,7 +333,7 @@ def ghci_dynamic( ):
return config.ghc_dynamic
def fast():
- return config.fast
+ return config.speed == 2
def platform( plat ):
return config.platform == plat
@@ -674,9 +674,10 @@ def test_common_work (name, opts, func, args):
# Which ways we are asked to skip
do_ways = list(filter (ok_way,all_ways))
- # In fast mode, we skip all but one way
- if config.fast and len(do_ways) > 0:
- do_ways = [do_ways[0]]
+ # Only run all ways in slow mode.
+ # See Note [validate and testsuite speed] in toplevel Makefile.
+ if config.speed > 0:
+ do_ways = do_ways[:1]
if not config.clean_only:
# Run the required tests...
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index 920d63433d..27d7f733af 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -247,10 +247,14 @@ else
set_list_broken =
endif
-ifeq "$(fast)" "YES"
-setfast = -e config.fast=1
+# See Note [validate and testsuite speed] in toplevel Makefile.
+ifneq "$(SPEED)" ""
+setspeed = -e config.speed="$(SPEED)"
+else ifeq "$(fast)" "YES"
+# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
+setspeed = -e config.speed=2
else
-setfast =
+setspeed =
endif
ifeq "$(accept)" "YES"
@@ -259,7 +263,7 @@ else
setaccept =
endif
-.PHONY: all boot test verbose accept fast list_broken
+.PHONY: all boot test verbose accept fast slow list_broken
all: test
@@ -278,7 +282,7 @@ test: $(TIMEOUT_PROGRAM)
$(patsubst %, --way=%, $(WAY)) \
$(patsubst %, --skipway=%, $(SKIPWAY)) \
$(set_list_broken) \
- $(setfast) \
+ $(setspeed) \
$(setaccept)
verbose: test
@@ -287,7 +291,11 @@ accept:
$(MAKE) accept=YES
fast:
- $(MAKE) fast=YES
+ # See Note [validate and testsuite speed] in toplevel Makefile.
+ $(MAKE) SPEED=2
+
+slow:
+ $(MAKE) SPEED=0
list_broken:
$(MAKE) list_broken=YES
diff --git a/testsuite/tests/concurrent/should_run/all.T b/testsuite/tests/concurrent/should_run/all.T
index f4bd78131c..58c3bd13e2 100644
--- a/testsuite/tests/concurrent/should_run/all.T
+++ b/testsuite/tests/concurrent/should_run/all.T
@@ -111,11 +111,7 @@ test('performGC', [ only_ways(['threaded1','threaded2'])
# -----------------------------------------------------------------------------
# These tests we only do for a full run
-def f( name, opts ):
- if config.fast:
- opts.skip = 1
-
-setTestOpts(f)
+setTestOpts(when(fast(), skip))
test('conc001', normal, compile_and_run, [''])
test('conc002', normal, compile_and_run, [''])
diff --git a/testsuite/tests/gadt/all.T b/testsuite/tests/gadt/all.T
index 9a48e69894..d5873c907f 100644
--- a/testsuite/tests/gadt/all.T
+++ b/testsuite/tests/gadt/all.T
@@ -52,7 +52,7 @@ test('arrow', normal, compile, [''])
test('tdpe', normal, compile, [''])
test('Nilsson', when(fast(), skip), compile, [''])
-if config.fast:
+if fast():
test('records', normal, compile, [''])
else:
test('records', normal, compile_and_run, [''])
diff --git a/testsuite/tests/typecheck/should_run/all.T b/testsuite/tests/typecheck/should_run/all.T
index 2433735a86..217f75e5a0 100755
--- a/testsuite/tests/typecheck/should_run/all.T
+++ b/testsuite/tests/typecheck/should_run/all.T
@@ -18,10 +18,7 @@ test('TcCoercible', when(compiler_lt('ghc', '7.7'), skip), compile_and_run, ['']
# -----------------------------------------------------------------------------
# Skip everything else if fast is on
-def f(name, opts):
- if config.fast:
- opts.skip = 1
-setTestOpts(f)
+setTestOpts(when(fast(), skip))
test('tcrun006', normal, compile_and_run, [''])
test('tcrun008', normal, compile_and_run, [''])
diff --git a/validate b/validate
index 5f24e19bd1..188605ddb4 100755
--- a/validate
+++ b/validate
@@ -18,9 +18,9 @@ Flags:
compiler the test suite covers.
2008-07-01: 63% slower than the default.
HTML generated here: testsuite/hpc_output/hpc_index.html
- --fast Omit dyn way, omit binary distribution
- --slow Build stage2 with -DDEBUG. Skips tests that call
- compiler_stats_num_field.
+ --fast Omit binary distribution. Omit certain tests.
+ --slow Build stage2 with -DDEBUG. Run tests for all WAYS,
+ but skip those that call compiler_stats_num_field.
2008-07-01: 14% slower than the default.
--dph Also build libraries/dph and run associated tests.
--quiet More pretty build log.
@@ -248,11 +248,11 @@ fi
case "$speed" in
SLOW)
- MAKE_TEST_TARGET=fulltest
+ MAKE_TEST_TARGET=slowtest
BINDIST="BINDIST=YES"
;;
NORMAL)
- MAKE_TEST_TARGET=fasttest
+ MAKE_TEST_TARGET=test
BINDIST="BINDIST=YES"
;;
FAST)