Make validate more quiet

  * By default use V=0, and call the testsuite with VERBOSE=2, which we
    did before only with validate --quiet. This disables printing the
    test commands it runs.

  * When --quiet is used, call the testsuite with VERBOSE=1. This
    disables printing the '====> Scanning' lines, and doesn't print
    which test is being run. So it only prints something when a test
    accidentally prints to stdout or when it fails.

    Don't set this option on Travis, as Travis will cancel a build if it
    doesn't see any output for more than 10 minutes.

  * When --quiet is used, set the new test option NO_PRINT_SUMMARY,
    which skips printing the test summary. Only the list of unexpected
    failures is printed, if there are any. Note that the full summary
    can still be found in testsuite_summary.txt

  * When --quiet is used, don't pass the `-v` flag to `ghc-pkg check`

  * When --quiet is used, don't print the Oops! header. It shoud be
    clear from the list of failing tests that something is wrong.

This is all done to get the most out of 30 lines of logfile. These changes can
be disabled later by simply not passing the --quiet flag to validate.

Differential Revision: https://phabricator.haskell.org/D942

7 files changed, 67 insertions, 42 deletions

diff --git a/.travis.yml b/.travis.yml
index d2e961dbb4..dd5a648eda 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -34,5 +34,6 @@ script:
  - echo 'DYNAMIC_GHC_PROGRAMS = NO' >> mk/validate.mk
  - echo 'GhcLibWays = v' >> mk/validate.mk
  - if [ "$DEBUG_STAGE" = "YES" ]; then echo 'GhcStage2HcOpts += -DDEBUG' >> mk/validate.mk; fi
- # use --quiet, otherwise we hit log file limits on travis.
- - CPUS=2 SKIP_PERF_TESTS=YES ./validate --fast --quiet
+ # Don't use --quiet, as it might cause the testsuite to not print output for
+ # over 10 minutes, causing Travis to kill our job.
+ - CPUS=2 SKIP_PERF_TESTS=YES ./validate --fast
diff --git a/Makefile b/Makefile
index e6e6a90601..f0f864dc83 100644
--- a/Makefile
+++ b/Makefile
@@ -156,8 +156,8 @@ endif
 
 .PHONY: fasttest
 fasttest:
-	$(MAKE) -C testsuite/tests CLEANUP=1 OUTPUT_SUMMARY=../../testsuite_summary.txt fast
+	$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast
 
 .PHONY: fulltest test
 fulltest test:
-	$(MAKE) -C testsuite/tests CLEANUP=1 OUTPUT_SUMMARY=../../testsuite_summary.txt
+	$(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 72e14192e9..87a0889c18 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -43,7 +43,8 @@ long_options = [
   "configfile=",	# config file
   "config=",  		# config field
   "rootdir=", 		# root of tree containing tests (default: .)
-  "output-summary=", 	# file in which to save the (human-readable) summary
+  "summary-file=",      # file in which to save the (human-readable) summary
+  "no-print-summary=",  # should we print the summary?
   "only=",		# just this test (can be give multiple --only= flags)
   "way=",		# just this way
   "skipway=",		# skip this way
@@ -71,8 +72,11 @@ for opt,arg in opts:
     if opt == '--rootdir':
         config.rootdirs.append(arg)
 
-    if opt == '--output-summary':
-        config.output_summary = arg
+    if opt == '--summary-file':
+        config.summary_file = arg
+
+    if opt == '--no-print-summary':
+        config.no_print_summary = True
 
     if opt == '--only':
         config.only.append(arg)
@@ -306,10 +310,10 @@ else:
             break
         oneTest()
         
-    summary(t, sys.stdout)
+    summary(t, sys.stdout, config.no_print_summary)
 
-    if config.output_summary != '':
-        summary(t, open(config.output_summary, 'w'))
+    if config.summary_file != '':
+        summary(t, open(config.summary_file, 'w'))
 
 sys.exit(0)
 
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 643dad60bc..4b6acfd9bf 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -29,7 +29,16 @@ class TestConfig:
         self.accept = 0
 
         # File in which to save the summary
-        self.output_summary = ''
+        self.summary_file = ''
+
+        # Should we print the summary?
+        # Disabling this is useful for Phabricator/Harbormaster
+        # logfiles, which are truncated to 30 lines. TODO. Revise if
+        # this is still true.
+        # Note that we have a separate flag for this, instead of
+        # overloading --verbose, as you might want to see the summary
+        # with --verbose=0.
+        self.no_print_summary = False
 
         # File in which to save the times
         self.times_file = ''
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 98a75e011e..fe9125b650 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -2186,10 +2186,15 @@ def findTFiles_(path):
 # -----------------------------------------------------------------------------
 # Output a test summary to the specified file object
 
-def summary(t, file):
+def summary(t, file, short=False):
 
     file.write('\n')
     printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures])
+
+    if short:
+        # Only print the list of unexpected tests above.
+        return
+
     file.write('OVERALL SUMMARY for test run started at '
                + time.strftime("%c %Z", t.start_time) + '\n'
                + str(datetime.timedelta(seconds=
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index 7e5b038e1a..5b482059f1 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -223,9 +223,13 @@ RUNTEST_OPTS +=  \
 	--config 'gs=$(call quote_path,$(GS))' \
 	--config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
 
-ifneq "$(OUTPUT_SUMMARY)" ""
+ifneq "$(SUMMARY_FILE)" ""
 RUNTEST_OPTS +=  \
-	--output-summary "$(OUTPUT_SUMMARY)"
+	--summary-file "$(SUMMARY_FILE)"
+endif
+ifeq "$(NO_PRINT_SUMMARY)" "YES"
+RUNTEST_OPTS +=  \
+	--no-print-summary 1
 endif
 
 RUNTEST_OPTS +=  \
diff --git a/validate b/validate
index 647bb45a03..fbf3c1be63 100755
--- a/validate
+++ b/validate
@@ -83,14 +83,21 @@ do
 done
 
 check_packages () {
-    echo "== Start $1 package check"
     if [ "$bindistdir" = "" ]
     then
-        inplace/bin/ghc-pkg check -v
+        ghc_pkg=inplace/bin/ghc-pkg
     else
-        "$bindistdir"/bin/ghc-pkg check -v
+        ghc_pkg="$bindistdir"/bin/ghc-pkg
+    fi
+
+    if [ $be_quiet -eq 1 ]
+    then
+        "$ghc_pkg" check
+    else
+        echo "== Start $1 package check"
+        "$ghc_pkg" check -v
+        echo "== End $1 package check"
     fi
-    echo "== End $1 package check"
 }
 
 detect_cpu_count () {
@@ -133,17 +140,9 @@ fi
 
 if type gmake > /dev/null 2> /dev/null
 then
-    if [ $be_quiet -eq 1 ]; then
-        make="gmake -s"
-    else
-        make="gmake"
-    fi
+    make="gmake -s"
 else
-    if [ $be_quiet -eq 1 ]; then
-        make="make -s"
-    else
-        make="make"
-    fi
+    make="make -s"
 fi
 
 if [ $testsuite_only -eq 0 ]; then
@@ -170,10 +169,7 @@ thisdir=`utils/ghc-pwd/dist-boot/ghc-pwd`
 echo "Validating=YES"       >  mk/are-validating.mk
 echo "ValidateSpeed=$speed" >> mk/are-validating.mk
 echo "ValidateHpc=$hpc"     >> mk/are-validating.mk
-
-if [ $be_quiet -eq 1 ]; then
-    echo "V=0"              >> mk/are-validating.mk # Less gunk
-fi
+echo "V=0"                  >> mk/are-validating.mk # Less gunk
 
 if [ $use_dph -eq 1 ]; then
     echo "BUILD_DPH=YES"    >> mk/are-validating.mk
@@ -238,12 +234,13 @@ FAST)
         ;;
 esac
 
-verbosity=3
 if [ $be_quiet -eq 1 ]; then
-   verbosity=2
+    TEST_VERBOSITY="VERBOSE=1 NO_PRINT_SUMMARY=YES"
+else
+    TEST_VERBOSITY="VERBOSE=2"
 fi
 
-$make $MAKE_TEST_TARGET stage=2 $BINDIST VERBOSE=$verbosity THREADS=$threads 2>&1 | tee testlog
+$make $MAKE_TEST_TARGET stage=2 $BINDIST $TEST_VERBOSITY THREADS=$threads 2>&1 | tee testlog
 
 check_packages post-testsuite
 
@@ -252,16 +249,16 @@ then
     utils/hpc/hpc markup --hpcdir=. --srcdir=compiler --srcdir=testsuite/hpc_output --destdir=testsuite/hpc_output testsuite/hpc_output/ghc.tix
 fi
 
-echo "-------------------------------------------------------------------"
 if
-    grep '\<0 caused framework failures' testlog >/dev/null 2>/dev/null &&
-    grep '\<0 unexpected passes' testlog >/dev/null 2>/dev/null &&
-    grep '\<0 unexpected failures' testlog >/dev/null 2>/dev/null &&
-    grep '\<0 unexpected stat failures' testlog >/dev/null 2>/dev/null &&
-    ! grep 'Some files are written by multiple tests' testlog >/dev/null 2>/dev/null ; then
+    grep '\<0 caused framework failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
+    grep '\<0 unexpected passes' testsuite_summary.txt >/dev/null 2>/dev/null &&
+    grep '\<0 unexpected failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
+    grep '\<0 unexpected stat failures' testsuite_summary.txt >/dev/null 2>/dev/null &&
+    ! grep 'Some files are written by multiple tests' testsuite_summary.txt >/dev/null 2>/dev/null ; then
     if [ $testsuite_only -eq 0 ] && [ $no_clean -eq 0 ]
     then
         cat <<EOF
+-------------------------------------------------------------------
 Congratulations!  This tree has passed minimal testing.
 
 NOTE: If you have made changes that may cause failures not tested for by
@@ -283,6 +280,7 @@ EOF
 EOF
     else
         cat <<EOF
+-------------------------------------------------------------------
 I didn't find any problems, but this wasn't a complete validate run,
 so be careful!
 
@@ -292,10 +290,14 @@ the minimal testing procedure, please do further testing as necessary.
 EOF
    fi
 else
-    cat <<EOF
+    if [ $be_quiet -eq 0 ]
+    then
+       cat <<EOF
+-------------------------------------------------------------------
 Oops!  Looks like you have some unexpected test results or framework failures.
 Please fix them before pushing/sending patches.
 -------------------------------------------------------------------
 EOF
+    fi
     exit 1
 fi