Testsuite: validate the tests/stage1 directory with the stage1 compiler

* See `Note [Why is there no stage1 setup function?]`.
* Move T2632 to the tests/stage1 directory (#10382).

Reviewed by: ezyang, nomeata, bgamari

Differential Revision: https://phabricator.haskell.org/D2341

GHC Trac Issues: #12197

10 files changed, 83 insertions, 12 deletions

diff --git a/.gitignore b/.gitignore
index 937d58ffd2..3c9a2fbcd1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -163,8 +163,8 @@ _darcs/
 /rts/package.conf.install
 /rts/package.conf.install.raw
 /stage3.package.conf
-/testsuite_summary.txt
-/testlog
+/testsuite_summary*.txt
+/testlog*
 /utils/runghc/runghc.cabal
 
 /extra-gcc-opts
diff --git a/testsuite/Makefile b/testsuite/Makefile
index b366dc3144..46182fc36f 100644
--- a/testsuite/Makefile
+++ b/testsuite/Makefile
@@ -28,10 +28,10 @@ verbose:
 accept:
 	$(MAKE) -C $(TOP)/tests accept
 
-fast:
+fast fasttest:
 	$(MAKE) -C $(TOP)/tests fast
 
-slow:
+slow slowtest:
 	$(MAKE) -C $(TOP)/tests slow
 
 list_broken:
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index d08141f251..fbe030a07a 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -60,11 +60,14 @@ class TestConfig:
 
         self.list_broken = False
 
-        # Path to the compiler
+        # Path to the compiler (stage2 by default)
         self.compiler = ''
         # and ghc-pkg
         self.ghc_pkg = ''
 
+        # Is self.compiler a stage 1, 2 or 3 compiler?
+        self.stage = 2
+
         # Compiler version info
         self.compiler_version = ''
         self.compiler_maj_version = ''
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index ced16d18f8..1c20936e60 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -100,6 +100,36 @@ def expect_fail( name, opts ):
 def reqlib( lib ):
     return lambda name, opts, l=lib: _reqlib (name, opts, l )
 
+def stage1(name, opts):
+    # See Note [Why is there no stage1 setup function?]
+    framework_fail(name, 'stage1 setup function does not exist',
+                   'add your test to testsuite/tests/stage1 instead')
+
+# Note [Why is there no stage1 setup function?]
+#
+# Presumably a stage1 setup function would signal that the stage1
+# compiler should be used to compile a test.
+#
+# Trouble is, the path to the compiler + the `ghc --info` settings for
+# that compiler are currently passed in from the `make` part of the
+# testsuite driver.
+#
+# Switching compilers in the Python part would be entirely too late, as
+# all ghc_with_* settings would be wrong. See config/ghc for possible
+# consequences (for example, config.run_ways would still be
+# based on the default compiler, quite likely causing ./validate --slow
+# to fail).
+#
+# It would be possible to let the Python part of the testsuite driver
+# make the call to `ghc --info`, but doing so would require quite some
+# work. Care has to be taken to not affect the run_command tests for
+# example, as they also use the `ghc --info` settings:
+#     quasiquotation/qq007/Makefile:ifeq "$(GhcDynamic)" "YES"
+#
+# If you want a test to run using the stage1 compiler, add it to the
+# testsuite/tests/stage1 directory. Validate runs the tests in that
+# directory with `make stage=1`.
+
 # Cache the results of looking to see if we have a library or not.
 # This makes quite a difference, especially on Windows.
 have_lib = {}
@@ -2038,7 +2068,7 @@ def summary(t, file, short=False):
         # Only print the list of unexpected tests above.
         return
 
-    file.write('OVERALL SUMMARY for test run started at '
+    file.write('SUMMARY for test run started at '
                + time.strftime("%c %Z", t.start_time) + '\n'
                + str(datetime.timedelta(seconds=
                     round(time.time() - time.mktime(t.start_time)))).rjust(8)
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index b5ea5a3ddd..92dc51ffb6 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -240,6 +240,8 @@ RUNTEST_OPTS +=  \
 	--config 'gs=$(call quote_path,$(GS))' \
 	--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
 
+RUNTEST_OPTS += -e "config.stage=$(GhcStage)"
+
 ifneq "$(SUMMARY_FILE)" ""
 RUNTEST_OPTS +=  \
 	--summary-file "$(SUMMARY_FILE)"
@@ -304,11 +306,11 @@ verbose: test
 accept:
 	$(MAKE) accept=YES
 
-fast:
+fast fasttest:
 	# See Note [validate and testsuite speed] in toplevel Makefile.
 	$(MAKE) SPEED=2
 
-slow:
+slow slowtest:
 	$(MAKE) SPEED=0
 
 list_broken:
diff --git a/testsuite/tests/quotes/all.T b/testsuite/tests/quotes/all.T
index 87081a5dc6..fa8bd4b634 100644
--- a/testsuite/tests/quotes/all.T
+++ b/testsuite/tests/quotes/all.T
@@ -3,7 +3,6 @@ def f(name, opts):
 
 setTestOpts(f)
 
-test('T2632', normal, compile, [''])
 test('T2931', normal, compile, ['-v0'])
 test('T3572', normal, compile_and_run, [''])
 test('T4056', normal, compile, ['-v0'])
diff --git a/testsuite/tests/stage1/Makefile b/testsuite/tests/stage1/Makefile
new file mode 100644
index 0000000000..0916e4b1db
--- /dev/null
+++ b/testsuite/tests/stage1/Makefile
@@ -0,0 +1,6 @@
+TOP=../..
+
+stage = 1
+
+include $(TOP)/mk/boilerplate.mk
+include $(TOP)/mk/test.mk
diff --git a/testsuite/tests/quotes/T2632.hs b/testsuite/tests/stage1/T2632.hs
index 71f6350cc2..71f6350cc2 100644
--- a/testsuite/tests/quotes/T2632.hs
+++ b/testsuite/tests/stage1/T2632.hs
diff --git a/testsuite/tests/stage1/all.T b/testsuite/tests/stage1/all.T
new file mode 100644
index 0000000000..a7f01b3525
--- /dev/null
+++ b/testsuite/tests/stage1/all.T
@@ -0,0 +1,6 @@
+
+setTestOpts(unless(config.stage == 1, skip))
+
+# Test with stage1 compiler, see ticket:10382#comment:20.
+test('T2632', normal, compile,
+     ['-XTemplateHaskellQuotes -package template-haskell'])
diff --git a/validate b/validate
index 3efaf3aeed..fc289e5d17 100755
--- a/validate
+++ b/validate
@@ -266,11 +266,31 @@ FAST)
 esac
 
 if [ $be_quiet -eq 1 ] && [ -z $VERBOSE ]; then
-    TEST_VERBOSITY="VERBOSE=1 NO_PRINT_SUMMARY=YES"
+    TEST_VERBOSITY="VERBOSE=1"
 fi
 
+rm testsuite_summary.txt testsuite_summary_stage1.txt
+
 # Use LOCAL=0, see Note [Running tests in /tmp].
-$make $MAKE_TEST_TARGET stage=2 $BINDIST LOCAL=0 $TEST_VERBOSITY THREADS=$threads 2>&1 | tee testlog
+$make -C testsuite/tests $BINDIST \
+      $MAKE_TEST_TARGET stage=2 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
+      NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../testsuite_summary.txt \
+      2>&1 | tee testlog
+
+# Run a few tests using the stage1 compiler.
+# See Note [Why is there no stage1 setup function?].
+# Don't use BINDIST=YES, as stage1 is not available in a bindist.
+$make -C testsuite/tests/stage1 \
+      $MAKE_TEST_TARGET stage=1 LOCAL=0 $TEST_VERBOSITY THREADS=$threads \
+      NO_PRINT_SUMMARY=YES SUMMARY_FILE=../../../testsuite_summary_stage1.txt \
+      2>&1 | tee testlog-stage1
+
+echo
+echo '==== STAGE 1 TESTS ==== '
+cat testsuite_summary_stage1.txt
+
+echo '==== STAGE 2 TESTS ==== '
+cat testsuite_summary.txt
 
 check_packages post-testsuite
 
@@ -284,7 +304,12 @@ if
     grep '\<0 unexpected passes' testsuite_summary.txt >/dev/null 2>/dev/null &&
     grep '\<0 unexpected failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
     grep '\<0 unexpected stat failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
-    ! grep 'Some files are written by multiple tests' testsuite_summary.txt >/dev/null 2>/dev/null ; then
+
+    grep '\<0 caused framework failures' testsuite_summary_stage1.txt >/dev/null 2>/dev/null &&
+    grep '\<0 unexpected passes' testsuite_summary_stage1.txt >/dev/null 2>/dev/null &&
+    grep '\<0 unexpected failures' testsuite_summary_stage1.txt >/dev/null 2>/dev/null &&
+    grep '\<0 unexpected stat failures' testsuite_summary_stage1.txt >/dev/null 2>/dev/null ; then
+
     if [ $testsuite_only -eq 0 ] && [ $no_clean -eq 0 ]
     then
         cat <<EOF