diff options
| -rw-r--r-- | .travis.yml | 5 | ||||
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | testsuite/driver/runtests.py | 16 | ||||
| -rw-r--r-- | testsuite/driver/testglobals.py | 11 | ||||
| -rw-r--r-- | testsuite/driver/testlib.py | 7 | ||||
| -rw-r--r-- | testsuite/mk/test.mk | 8 | ||||
| -rwxr-xr-x | validate | 58 |
7 files changed, 67 insertions, 42 deletions
diff --git a/.travis.yml b/.travis.yml index d2e961dbb4..dd5a648eda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,5 +34,6 @@ script: - echo 'DYNAMIC_GHC_PROGRAMS = NO' >> mk/validate.mk - echo 'GhcLibWays = v' >> mk/validate.mk - if [ "$DEBUG_STAGE" = "YES" ]; then echo 'GhcStage2HcOpts += -DDEBUG' >> mk/validate.mk; fi - # use --quiet, otherwise we hit log file limits on travis. - - CPUS=2 SKIP_PERF_TESTS=YES ./validate --fast --quiet + # Don't use --quiet, as it might cause the testsuite to not print output for + # over 10 minutes, causing Travis to kill our job. + - CPUS=2 SKIP_PERF_TESTS=YES ./validate --fast @@ -156,8 +156,8 @@ endif .PHONY: fasttest fasttest: - $(MAKE) -C testsuite/tests CLEANUP=1 OUTPUT_SUMMARY=../../testsuite_summary.txt fast + $(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt fast .PHONY: fulltest test fulltest test: - $(MAKE) -C testsuite/tests CLEANUP=1 OUTPUT_SUMMARY=../../testsuite_summary.txt + $(MAKE) -C testsuite/tests CLEANUP=1 SUMMARY_FILE=../../testsuite_summary.txt diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py index 72e14192e9..87a0889c18 100644 --- a/testsuite/driver/runtests.py +++ b/testsuite/driver/runtests.py @@ -43,7 +43,8 @@ long_options = [ "configfile=", # config file "config=", # config field "rootdir=", # root of tree containing tests (default: .) - "output-summary=", # file in which to save the (human-readable) summary + "summary-file=", # file in which to save the (human-readable) summary + "no-print-summary=", # should we print the summary? "only=", # just this test (can be give multiple --only= flags) "way=", # just this way "skipway=", # skip this way @@ -71,8 +72,11 @@ for opt,arg in opts: if opt == '--rootdir': config.rootdirs.append(arg) - if opt == '--output-summary': - config.output_summary = arg + if opt == '--summary-file': + config.summary_file = arg + + if opt == '--no-print-summary': + config.no_print_summary = True if opt == '--only': config.only.append(arg) @@ -306,10 +310,10 @@ else: break oneTest() - summary(t, sys.stdout) + summary(t, sys.stdout, config.no_print_summary) - if config.output_summary != '': - summary(t, open(config.output_summary, 'w')) + if config.summary_file != '': + summary(t, open(config.summary_file, 'w')) sys.exit(0) diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py index 643dad60bc..4b6acfd9bf 100644 --- a/testsuite/driver/testglobals.py +++ b/testsuite/driver/testglobals.py @@ -29,7 +29,16 @@ class TestConfig: self.accept = 0 # File in which to save the summary - self.output_summary = '' + self.summary_file = '' + + # Should we print the summary? + # Disabling this is useful for Phabricator/Harbormaster + # logfiles, which are truncated to 30 lines. TODO. Revise if + # this is still true. + # Note that we have a separate flag for this, instead of + # overloading --verbose, as you might want to see the summary + # with --verbose=0. + self.no_print_summary = False # File in which to save the times self.times_file = '' diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index 98a75e011e..fe9125b650 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -2186,10 +2186,15 @@ def findTFiles_(path): # ----------------------------------------------------------------------------- # Output a test summary to the specified file object -def summary(t, file): +def summary(t, file, short=False): file.write('\n') printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures]) + + if short: + # Only print the list of unexpected tests above. + return + file.write('OVERALL SUMMARY for test run started at ' + time.strftime("%c %Z", t.start_time) + '\n' + str(datetime.timedelta(seconds= diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk index 7e5b038e1a..5b482059f1 100644 --- a/testsuite/mk/test.mk +++ b/testsuite/mk/test.mk @@ -223,9 +223,13 @@ RUNTEST_OPTS += \ --config 'gs=$(call quote_path,$(GS))' \ --config 'timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))' -ifneq "$(OUTPUT_SUMMARY)" "" +ifneq "$(SUMMARY_FILE)" "" RUNTEST_OPTS += \ - --output-summary "$(OUTPUT_SUMMARY)" + --summary-file "$(SUMMARY_FILE)" +endif +ifeq "$(NO_PRINT_SUMMARY)" "YES" +RUNTEST_OPTS += \ + --no-print-summary 1 endif RUNTEST_OPTS += \ @@ -83,14 +83,21 @@ do done check_packages () { - echo "== Start $1 package check" if [ "$bindistdir" = "" ] then - inplace/bin/ghc-pkg check -v + ghc_pkg=inplace/bin/ghc-pkg else - "$bindistdir"/bin/ghc-pkg check -v + ghc_pkg="$bindistdir"/bin/ghc-pkg + fi + + if [ $be_quiet -eq 1 ] + then + "$ghc_pkg" check + else + echo "== Start $1 package check" + "$ghc_pkg" check -v + echo "== End $1 package check" fi - echo "== End $1 package check" } detect_cpu_count () { @@ -133,17 +140,9 @@ fi if type gmake > /dev/null 2> /dev/null then - if [ $be_quiet -eq 1 ]; then - make="gmake -s" - else - make="gmake" - fi + make="gmake -s" else - if [ $be_quiet -eq 1 ]; then - make="make -s" - else - make="make" - fi + make="make -s" fi if [ $testsuite_only -eq 0 ]; then @@ -170,10 +169,7 @@ thisdir=`utils/ghc-pwd/dist-boot/ghc-pwd` echo "Validating=YES" > mk/are-validating.mk echo "ValidateSpeed=$speed" >> mk/are-validating.mk echo "ValidateHpc=$hpc" >> mk/are-validating.mk - -if [ $be_quiet -eq 1 ]; then - echo "V=0" >> mk/are-validating.mk # Less gunk -fi +echo "V=0" >> mk/are-validating.mk # Less gunk if [ $use_dph -eq 1 ]; then echo "BUILD_DPH=YES" >> mk/are-validating.mk @@ -238,12 +234,13 @@ FAST) ;; esac -verbosity=3 if [ $be_quiet -eq 1 ]; then - verbosity=2 + TEST_VERBOSITY="VERBOSE=1 NO_PRINT_SUMMARY=YES" +else + TEST_VERBOSITY="VERBOSE=2" fi -$make $MAKE_TEST_TARGET stage=2 $BINDIST VERBOSE=$verbosity THREADS=$threads 2>&1 | tee testlog +$make $MAKE_TEST_TARGET stage=2 $BINDIST $TEST_VERBOSITY THREADS=$threads 2>&1 | tee testlog check_packages post-testsuite @@ -252,16 +249,16 @@ then utils/hpc/hpc markup --hpcdir=. --srcdir=compiler --srcdir=testsuite/hpc_output --destdir=testsuite/hpc_output testsuite/hpc_output/ghc.tix fi -echo "-------------------------------------------------------------------" if - grep '\<0 caused framework failures' testlog >/dev/null 2>/dev/null && - grep '\<0 unexpected passes' testlog >/dev/null 2>/dev/null && - grep '\<0 unexpected failures' testlog >/dev/null 2>/dev/null && - grep '\<0 unexpected stat failures' testlog >/dev/null 2>/dev/null && - ! grep 'Some files are written by multiple tests' testlog >/dev/null 2>/dev/null ; then + grep '\<0 caused framework failures' testsuite_summary.txt >/dev/null 2>/dev/null && + grep '\<0 unexpected passes' testsuite_summary.txt >/dev/null 2>/dev/null && + grep '\<0 unexpected failures' testsuite_summary.txt >/dev/null 2>/dev/null && + grep '\<0 unexpected stat failures' testsuite_summary.txt >/dev/null 2>/dev/null && + ! grep 'Some files are written by multiple tests' testsuite_summary.txt >/dev/null 2>/dev/null ; then if [ $testsuite_only -eq 0 ] && [ $no_clean -eq 0 ] then cat <<EOF +------------------------------------------------------------------- Congratulations! This tree has passed minimal testing. NOTE: If you have made changes that may cause failures not tested for by @@ -283,6 +280,7 @@ EOF EOF else cat <<EOF +------------------------------------------------------------------- I didn't find any problems, but this wasn't a complete validate run, so be careful! @@ -292,10 +290,14 @@ the minimal testing procedure, please do further testing as necessary. EOF fi else - cat <<EOF + if [ $be_quiet -eq 0 ] + then + cat <<EOF +------------------------------------------------------------------- Oops! Looks like you have some unexpected test results or framework failures. Please fix them before pushing/sending patches. ------------------------------------------------------------------- EOF + fi exit 1 fi |