testsuite: handle missing stats files gracefully (#10305)

The following tests would result in framework failures when using a ghc
build with HADDOCK_DOCS=NO in mk/build.mk or mk/validate.mk:

  * haddock.Cabal
  * haddock.base
  * haddock.compiler

Test Plan: run make in tests/perf/haddock

Differential Revision: https://phabricator.haskell.org/D899

(cherry picked from commit 6694ccf9444baf565eb0f38f7808767616f23825)

5 files changed, 31 insertions, 11 deletions

diff --git a/testsuite/config/ghc b/testsuite/config/ghc
index c208838268..bf1bc77e70 100644
--- a/testsuite/config/ghc
+++ b/testsuite/config/ghc
@@ -9,6 +9,7 @@ config.compiler_type         = 'ghc'
 config.compiler              = 'ghc'
 config.compiler_always_flags = ghc_compiler_always_flags.split()
 
+config.haddock               = 'haddock'
 config.hp2ps                 = 'hp2ps'
 config.hpc                   = 'hpc'
 config.gs                    = 'gs'
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index af1dcdf675..65ff8bab0e 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -123,6 +123,10 @@ def _reqlib( name, opts, lib ):
     if not got_it:
         opts.expect = 'missing-lib'
 
+def req_haddock( name, opts ):
+    if not config.haddock:
+        opts.expect = 'missing-lib'
+
 def req_profiling( name, opts ):
     if not config.have_profiling:
         opts.expect = 'fail'
@@ -1115,7 +1119,10 @@ def checkStats(name, way, stats_file, range_fields):
 
     result = passed()
     if len(range_fields) > 0:
-        f = open(in_testdir(stats_file))
+        try:
+            f = open(in_testdir(stats_file))
+        except IOError as e:
+            return failBecause(str(e))
         contents = f.read()
         f.close()
 
diff --git a/testsuite/mk/boilerplate.mk b/testsuite/mk/boilerplate.mk
index 1765d78f03..98c9886f44 100644
--- a/testsuite/mk/boilerplate.mk
+++ b/testsuite/mk/boilerplate.mk
@@ -95,6 +95,10 @@ ifeq "$(RUNGHC)" ""
 RUNGHC := $(BIN_ROOT)/runghc
 endif
 
+ifeq "$(HADDOCK)" ""
+HADDOCK := $(call find_tool,haddock)
+endif
+
 ifeq "$(HSC2HS)" ""
 HSC2HS := $(BIN_ROOT)/hsc2hs
 endif
@@ -117,6 +121,12 @@ ifeq "$(shell test -x '$(GHC_PKG)' && echo exists)" ""
 $(error Cannot find ghc-pkg: $(GHC_PKG))
 endif
 
+$(eval $(call canonicaliseExecutable,HADDOCK))
+ifeq "$(shell test -x '$(HADDOCK)' && echo exists)" ""
+# haddock is optional.
+HADDOCK :=
+endif
+
 $(eval $(call canonicaliseExecutable,HSC2HS))
 ifeq "$(shell test -x '$(HSC2HS)' && echo exists)" ""
 $(error Cannot find hsc2hs: $(HSC2HS))
diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk
index 813a3a1da1..42e634a9d8 100644
--- a/testsuite/mk/test.mk
+++ b/testsuite/mk/test.mk
@@ -205,15 +205,17 @@ RUNTEST_OPTS +=  \
 	-e 'config.exeext="$(exeext)"' \
 	-e 'config.top="$(TOP_ABS)"'
 
-# Put an extra pair of quotes around program paths,
+# Put an extra pair of quotes around non-empty program paths,
 # so we don't have to in .T scripts or driver/testlib.py.
+quote_path = $(if $1,"\"$1\"","")
 RUNTEST_OPTS +=  \
-	-e 'config.compiler="\"$(TEST_HC)\""' \
-	-e 'config.ghc_pkg="\"$(GHC_PKG)\""' \
-	-e 'config.hp2ps="\"$(HP2PS_ABS)\""' \
-	-e 'config.hpc="\"$(HPC)\""' \
-	-e 'config.gs="\"$(GS)\""' \
-	-e 'config.timeout_prog="\"$(TIMEOUT_PROGRAM)\""'
+	-e 'config.compiler=$(call quote_path,$(TEST_HC))' \
+	-e 'config.ghc_pkg=$(call quote_path,$(GHC_PKG))' \
+	-e 'config.haddock=$(call quote_path,$(HADDOCK))' \
+	-e 'config.hp2ps=$(call quote_path,$(HP2PS_ABS))' \
+	-e 'config.hpc=$(call quote_path,$(HPC))' \
+	-e 'config.gs=$(call quote_path,$(GS))' \
+	-e 'config.timeout_prog=$(call quote_path,$(TIMEOUT_PROGRAM))'
 
 ifneq "$(OUTPUT_SUMMARY)" ""
 RUNTEST_OPTS +=  \
diff --git a/testsuite/tests/perf/haddock/all.T b/testsuite/tests/perf/haddock/all.T
index 905ab91a50..8e0b971c23 100644
--- a/testsuite/tests/perf/haddock/all.T
+++ b/testsuite/tests/perf/haddock/all.T
@@ -3,7 +3,7 @@
 # detect outliers, as described in Note [residency]. See #9556.
 
 test('haddock.base',
-     [unless(in_tree_compiler(), skip)
+     [unless(in_tree_compiler(), skip), req_haddock
      ,stats_num_field('bytes allocated',
           [(wordsize(64), 9502647104,  5)
             # 2012-08-14: 5920822352 (amd64/Linux)
@@ -40,7 +40,7 @@ test('haddock.base',
      ['../../../../libraries/base/dist-install/doc/html/base/base.haddock.t'])
 
 test('haddock.Cabal',
-     [unless(in_tree_compiler(), skip)
+     [unless(in_tree_compiler(), skip), req_haddock
      ,stats_num_field('bytes allocated',
           [(wordsize(64), 6387320816, 5)
             # 2012-08-14: 3255435248 (amd64/Linux)
@@ -79,7 +79,7 @@ test('haddock.Cabal',
      ['../../../../libraries/Cabal/Cabal/dist-install/doc/html/Cabal/Cabal.haddock.t'])
 
 test('haddock.compiler',
-     [unless(in_tree_compiler(), skip)
+     [unless(in_tree_compiler(), skip), req_haddock
      ,stats_num_field('bytes allocated',
           [(wordsize(64), 33562468736, 10)
             # 2012-08-14: 26070600504 (amd64/Linux)