Testsuite: by default run all tests for a single way

`make test` now runs all tests for a single way only. Use `make slowtest` to
get the previous behaviour (i.e. run all tests for all ways).

The intention is to use this new `make test` setting for Phabricator, as
a reasonable compromise between `make fasttest` (what it previously
used) and a fullblown `make slowtest` (which runs all tests for all
ways).

See Note [validate and testsuite speed] in toplevel Makefile.

Differential Revision: https://phabricator.haskell.org/D1178

10 files changed, 64 insertions, 30 deletions

diff --git a/.travis.yml b/.travis.yml
index 660d38305a..3a9d337344 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -53,4 +53,6 @@ script:
  # * But use VERBOSE=2 (the default, but not when using --quiet) otherwise
  # the testsuite might not print output for over 10 minutes (more likely so
  # when DEBUG_STAGE2=NO), causing Travis to again kill our job.
+ # * Use --fast, to stay within the time limits set by Travis.
+ # See Note [validate and testsuite speed] in toplevel Makefile.
  - THREADS=3 SKIP_PERF_TESTS=YES VERBOSE=2 ./validate --fast --quiet
diff --git a/Makefile b/Makefile
index b1b1742eb3..e6a1bc5b02 100644
--- a/Makefile
+++ b/Makefile
@@ -83,6 +83,7 @@ REALGOALS=$(filter-out \
     help \
     test \
     fulltest \
+    slowtest \
     fasttest \
   ,$(MAKECMDGOALS))
 
@@ -159,10 +160,36 @@ endif
 
 endif
 
+# Note [validate and testsuite speed]
+#
+# There are 3 different validate and testsuite speed settings:
+# fast, normal and slow.
+#
+#                              how    how   used
+#           cd tests  config.  many   many  by
+# validate  && make   speed=   tests  ways  whom
+# =============================================================================
+# --fast    fast      2        some   1     Travis (to stay within time limit)
+# --normal  test      1        all    1     Phabricator (slow takes too long?)
+# --slow    slow      0        all    all   Nightly (slow is ok)
+#
+#           accept    1        all    1
+#
+# `make accept` should run all tests exactly once. There is no point in
+# accepting a test for multiple ways, since it should produce the same output
+# for all ways.
+#
+# To make sure all .stderr and .stdout files in the testsuite are never
+# out-of-date, it is useful if Phabricator, via a normal `./validate` and `make
+# test`, runs each test at least once.
 .PHONY: fasttest
 fasttest:
 	$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast
 
-.PHONY: fulltest test
-fulltest test:
+.PHONY: test
+test:
 	$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt
+
+.PHONY: slowtest fulltest
+slowtest fulltest:
+	$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt slow
diff --git a/testsuite/Makefile b/testsuite/Makefile
index 6ad34393fc..661769b891 100644
--- a/testsuite/Makefile
+++ b/testsuite/Makefile
@@ -11,7 +11,7 @@ ifneq "$(MAKECMDGOALS)" "maintainer-clean"
 
 include $(TOP)/mk/boilerplate.mk
 
-.PHONY: all boot test verbose accept fast list_broken
+.PHONY: all boot test verbose accept fast slow list_broken
 
 boot:
 	$(MAKE) -C $(TOP)/timeout all
@@ -31,6 +31,9 @@ accept:
 fast:
 	$(MAKE) -C $(TOP)/tests fast
 
+slow:
+	$(MAKE) -C $(TOP)/tests slow
+
 list_broken:
 	$(MAKE) -C $(TOP)/tests list_broken
 
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 4b6754fa40..95168f3c5e 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -54,8 +54,8 @@ class TestConfig:
         # Verbosity level
         self.verbose = 3
 
-        # run the "fast" version of the test suite
-        self.fast = 0
+        # See Note [validate and testsuite speed] in toplevel Makefile.
+        self.speed = 1
 
         self.list_broken = False
 
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index c6a3bc2461..68a9208a4b 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -333,7 +333,7 @@ def ghci_dynamic( ):
     return config.ghc_dynamic
 
 def fast():
-    return config.fast
+    return config.speed == 2
 
 def platform( plat ):
     return config.platform == plat
@@ -674,9 +674,10 @@ def test_common_work (name, opts, func, args):
         # Which ways we are asked to skip
         do_ways = list(filter (ok_way,all_ways))
 
-        # In fast mode, we skip all but one way
-        if config.fast and len(do_ways) > 0:
-            do_ways = [do_ways[0]]
+        # Only run all ways in slow mode.
+        # See Note [validate and testsuite speed] in toplevel Makefile.
+        if config.speed > 0:
+            do_ways = do_ways[:1]
 
         if not config.clean_only:
             # Run the required tests...
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index 920d63433d..27d7f733af 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -247,10 +247,14 @@ else
 set_list_broken = 
 endif
 
-ifeq "$(fast)" "YES"
-setfast = -e config.fast=1
+# See Note [validate and testsuite speed] in toplevel Makefile.
+ifneq "$(SPEED)" ""
+setspeed = -e config.speed="$(SPEED)"
+else ifeq "$(fast)" "YES"
+# Backward compatibility. Maybe some people are running 'make accept fast=YES'?
+setspeed = -e config.speed=2
 else
-setfast = 
+setspeed =
 endif
 
 ifeq "$(accept)" "YES"
@@ -259,7 +263,7 @@ else
 setaccept = 
 endif
 
-.PHONY: all boot test verbose accept fast list_broken
+.PHONY: all boot test verbose accept fast slow list_broken
 
 all: test
 
@@ -278,7 +282,7 @@ test: $(TIMEOUT_PROGRAM)
 		$(patsubst %, --way=%, $(WAY)) \
 		$(patsubst %, --skipway=%, $(SKIPWAY)) \
 		$(set_list_broken) \
-		$(setfast) \
+		$(setspeed) \
 		$(setaccept)
 
 verbose: test
@@ -287,7 +291,11 @@ accept:
 	$(MAKE) accept=YES
 
 fast:
-	$(MAKE) fast=YES
+	# See Note [validate and testsuite speed] in toplevel Makefile.
+	$(MAKE) SPEED=2
+
+slow:
+	$(MAKE) SPEED=0
 
 list_broken:
 	$(MAKE) list_broken=YES
diff --git a/testsuite/tests/concurrent/should_run/all.T b/testsuite/tests/concurrent/should_run/all.T
index f4bd78131c..58c3bd13e2 100644
--- a/testsuite/tests/concurrent/should_run/all.T
+++ b/testsuite/tests/concurrent/should_run/all.T
@@ -111,11 +111,7 @@ test('performGC', [ only_ways(['threaded1','threaded2'])
 # -----------------------------------------------------------------------------
 # These tests we only do for a full run
 
-def f( name, opts ):
-  if config.fast:
-     opts.skip = 1
-
-setTestOpts(f)
+setTestOpts(when(fast(), skip))
 
 test('conc001', normal, compile_and_run, [''])
 test('conc002', normal, compile_and_run, [''])
diff --git a/testsuite/tests/gadt/all.T b/testsuite/tests/gadt/all.T
index 9a48e69894..d5873c907f 100644
--- a/testsuite/tests/gadt/all.T
+++ b/testsuite/tests/gadt/all.T
@@ -52,7 +52,7 @@ test('arrow', normal, compile, [''])
 test('tdpe', normal, compile, [''])
 test('Nilsson', when(fast(), skip), compile, [''])
 
-if config.fast:
+if fast():
     test('records', normal, compile, [''])
 else:
     test('records', normal, compile_and_run, [''])
diff --git a/testsuite/tests/typecheck/should_run/all.T b/testsuite/tests/typecheck/should_run/all.T
index 2433735a86..217f75e5a0 100755
--- a/testsuite/tests/typecheck/should_run/all.T
+++ b/testsuite/tests/typecheck/should_run/all.T
@@ -18,10 +18,7 @@ test('TcCoercible', when(compiler_lt('ghc', '7.7'), skip), compile_and_run, ['']
 # -----------------------------------------------------------------------------
 # Skip everything else if fast is on
 
-def f(name, opts):
-    if config.fast:
-        opts.skip = 1
-setTestOpts(f)
+setTestOpts(when(fast(), skip))
 
 test('tcrun006', normal, compile_and_run, [''])
 test('tcrun008', normal, compile_and_run, [''])
diff --git a/validate b/validate
index 5f24e19bd1..188605ddb4 100755
--- a/validate
+++ b/validate
@@ -18,9 +18,9 @@ Flags:
                     compiler the test suite covers.
                     2008-07-01: 63% slower than the default.
                     HTML generated here: testsuite/hpc_output/hpc_index.html
-  --fast            Omit dyn way, omit binary distribution
-  --slow            Build stage2 with -DDEBUG. Skips tests that call
-                    compiler_stats_num_field.
+  --fast            Omit binary distribution. Omit certain tests.
+  --slow            Build stage2 with -DDEBUG. Run tests for all WAYS,
+                    but skip those that call compiler_stats_num_field.
                     2008-07-01: 14% slower than the default.
   --dph             Also build libraries/dph and run associated tests.
   --quiet           More pretty build log.
@@ -248,11 +248,11 @@ fi
 
 case "$speed" in
 SLOW)
-        MAKE_TEST_TARGET=fulltest
+        MAKE_TEST_TARGET=slowtest
         BINDIST="BINDIST=YES"
         ;;
 NORMAL)
-        MAKE_TEST_TARGET=fasttest
+        MAKE_TEST_TARGET=test
         BINDIST="BINDIST=YES"
         ;;
 FAST)