diff options
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rw-r--r-- | Makefile | 31 | ||||
| -rw-r--r-- | testsuite/Makefile | 5 | ||||
| -rw-r--r-- | testsuite/driver/testglobals.py | 4 | ||||
| -rw-r--r-- | testsuite/driver/testlib.py | 9 | ||||
| -rw-r--r-- | testsuite/mk/test.mk | 20 | ||||
| -rw-r--r-- | testsuite/tests/concurrent/should_run/all.T | 6 | ||||
| -rw-r--r-- | testsuite/tests/gadt/all.T | 2 | ||||
| -rwxr-xr-x | testsuite/tests/typecheck/should_run/all.T | 5 | ||||
| -rwxr-xr-x | validate | 10 |
10 files changed, 64 insertions, 30 deletions
diff --git a/.travis.yml b/.travis.yml index 660d38305a..3a9d337344 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,4 +53,6 @@ script: # * But use VERBOSE=2 (the default, but not when using --quiet) otherwise # the testsuite might not print output for over 10 minutes (more likely so # when DEBUG_STAGE2=NO), causing Travis to again kill our job. + # * Use --fast, to stay within the time limits set by Travis. + # See Note [validate and testsuite speed] in toplevel Makefile. - THREADS=3 SKIP_PERF_TESTS=YES VERBOSE=2 ./validate --fast --quiet @@ -83,6 +83,7 @@ REALGOALS=$(filter-out \ help \ test \ fulltest \ + slowtest \ fasttest \ ,$(MAKECMDGOALS)) @@ -159,10 +160,36 @@ endif endif +# Note [validate and testsuite speed] +# +# There are 3 different validate and testsuite speed settings: +# fast, normal and slow. +# +# how how used +# cd tests config. many many by +# validate && make speed= tests ways whom +# ============================================================================= +# --fast fast 2 some 1 Travis (to stay within time limit) +# --normal test 1 all 1 Phabricator (slow takes too long?) +# --slow slow 0 all all Nightly (slow is ok) +# +# accept 1 all 1 +# +# `make accept` should run all tests exactly once. There is no point in +# accepting a test for multiple ways, since it should produce the same output +# for all ways. +# +# To make sure all .stderr and .stdout files in the testsuite are never +# out-of-date, it is useful if Phabricator, via a normal `./validate` and `make +# test`, runs each test at least once. .PHONY: fasttest fasttest: $(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast -.PHONY: fulltest test -fulltest test: +.PHONY: test +test: $(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt + +.PHONY: slowtest fulltest +slowtest fulltest: + $(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt slow diff --git a/testsuite/Makefile b/testsuite/Makefile index 6ad34393fc..661769b891 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -11,7 +11,7 @@ ifneq "$(MAKECMDGOALS)" "maintainer-clean" include $(TOP)/mk/boilerplate.mk -.PHONY: all boot test verbose accept fast list_broken +.PHONY: all boot test verbose accept fast slow list_broken boot: $(MAKE) -C $(TOP)/timeout all @@ -31,6 +31,9 @@ accept: fast: $(MAKE) -C $(TOP)/tests fast +slow: + $(MAKE) -C $(TOP)/tests slow + list_broken: $(MAKE) -C $(TOP)/tests list_broken diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py index 4b6754fa40..95168f3c5e 100644 --- a/testsuite/driver/testglobals.py +++ b/testsuite/driver/testglobals.py @@ -54,8 +54,8 @@ class TestConfig: # Verbosity level self.verbose = 3 - # run the "fast" version of the test suite - self.fast = 0 + # See Note [validate and testsuite speed] in toplevel Makefile. + self.speed = 1 self.list_broken = False diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index c6a3bc2461..68a9208a4b 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -333,7 +333,7 @@ def ghci_dynamic( ): return config.ghc_dynamic def fast(): - return config.fast + return config.speed == 2 def platform( plat ): return config.platform == plat @@ -674,9 +674,10 @@ def test_common_work (name, opts, func, args): # Which ways we are asked to skip do_ways = list(filter (ok_way,all_ways)) - # In fast mode, we skip all but one way - if config.fast and len(do_ways) > 0: - do_ways = [do_ways[0]] + # Only run all ways in slow mode. + # See Note [validate and testsuite speed] in toplevel Makefile. + if config.speed > 0: + do_ways = do_ways[:1] if not config.clean_only: # Run the required tests... diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk index 920d63433d..27d7f733af 100644 --- a/testsuite/mk/test.mk +++ b/testsuite/mk/test.mk @@ -247,10 +247,14 @@ else set_list_broken = endif -ifeq "$(fast)" "YES" -setfast = -e config.fast=1 +# See Note [validate and testsuite speed] in toplevel Makefile. +ifneq "$(SPEED)" "" +setspeed = -e config.speed="$(SPEED)" +else ifeq "$(fast)" "YES" +# Backward compatibility. Maybe some people are running 'make accept fast=YES'? +setspeed = -e config.speed=2 else -setfast = +setspeed = endif ifeq "$(accept)" "YES" @@ -259,7 +263,7 @@ else setaccept = endif -.PHONY: all boot test verbose accept fast list_broken +.PHONY: all boot test verbose accept fast slow list_broken all: test @@ -278,7 +282,7 @@ test: $(TIMEOUT_PROGRAM) $(patsubst %, --way=%, $(WAY)) \ $(patsubst %, --skipway=%, $(SKIPWAY)) \ $(set_list_broken) \ - $(setfast) \ + $(setspeed) \ $(setaccept) verbose: test @@ -287,7 +291,11 @@ accept: $(MAKE) accept=YES fast: - $(MAKE) fast=YES + # See Note [validate and testsuite speed] in toplevel Makefile. + $(MAKE) SPEED=2 + +slow: + $(MAKE) SPEED=0 list_broken: $(MAKE) list_broken=YES diff --git a/testsuite/tests/concurrent/should_run/all.T b/testsuite/tests/concurrent/should_run/all.T index f4bd78131c..58c3bd13e2 100644 --- a/testsuite/tests/concurrent/should_run/all.T +++ b/testsuite/tests/concurrent/should_run/all.T @@ -111,11 +111,7 @@ test('performGC', [ only_ways(['threaded1','threaded2']) # ----------------------------------------------------------------------------- # These tests we only do for a full run -def f( name, opts ): - if config.fast: - opts.skip = 1 - -setTestOpts(f) +setTestOpts(when(fast(), skip)) test('conc001', normal, compile_and_run, ['']) test('conc002', normal, compile_and_run, ['']) diff --git a/testsuite/tests/gadt/all.T b/testsuite/tests/gadt/all.T index 9a48e69894..d5873c907f 100644 --- a/testsuite/tests/gadt/all.T +++ b/testsuite/tests/gadt/all.T @@ -52,7 +52,7 @@ test('arrow', normal, compile, ['']) test('tdpe', normal, compile, ['']) test('Nilsson', when(fast(), skip), compile, ['']) -if config.fast: +if fast(): test('records', normal, compile, ['']) else: test('records', normal, compile_and_run, ['']) diff --git a/testsuite/tests/typecheck/should_run/all.T b/testsuite/tests/typecheck/should_run/all.T index 2433735a86..217f75e5a0 100755 --- a/testsuite/tests/typecheck/should_run/all.T +++ b/testsuite/tests/typecheck/should_run/all.T @@ -18,10 +18,7 @@ test('TcCoercible', when(compiler_lt('ghc', '7.7'), skip), compile_and_run, [''] # ----------------------------------------------------------------------------- # Skip everything else if fast is on -def f(name, opts): - if config.fast: - opts.skip = 1 -setTestOpts(f) +setTestOpts(when(fast(), skip)) test('tcrun006', normal, compile_and_run, ['']) test('tcrun008', normal, compile_and_run, ['']) @@ -18,9 +18,9 @@ Flags: compiler the test suite covers. 2008-07-01: 63% slower than the default. HTML generated here: testsuite/hpc_output/hpc_index.html - --fast Omit dyn way, omit binary distribution - --slow Build stage2 with -DDEBUG. Skips tests that call - compiler_stats_num_field. + --fast Omit binary distribution. Omit certain tests. + --slow Build stage2 with -DDEBUG. Run tests for all WAYS, + but skip those that call compiler_stats_num_field. 2008-07-01: 14% slower than the default. --dph Also build libraries/dph and run associated tests. --quiet More pretty build log. @@ -248,11 +248,11 @@ fi case "$speed" in SLOW) - MAKE_TEST_TARGET=fulltest + MAKE_TEST_TARGET=slowtest BINDIST="BINDIST=YES" ;; NORMAL) - MAKE_TEST_TARGET=fasttest + MAKE_TEST_TARGET=test BINDIST="BINDIST=YES" ;; FAST) |