testsuite: introduce 'static_stats' tests

They are a particular type of perf tests. This patch introduces a
'stats_files_dir' configuration field in the testsuite driver where all
haddock timing files (and possibly others in the future) are assumed to live.
We also change both the Make and Hadrian build systems to pass respectively
$(TOP)/testsuite/tests/perf/haddock/ and
<build root>/stage1/haddock-timing-files/ as the value of that new
configuration field, and to generate the timing files in those directories
in the first place while generating documentation with haddock.

This new test type can be seen as one dedicated to examining stats files that
are generated while building a GHC distribution. This also lets us get rid of
the 'extra_files' directives in the all.T entries for haddock.base,
haddock.Cabal and haddock.compiler.

2 files changed, 16 insertions, 5 deletions

diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index b28477c769..ababefba19 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -139,6 +139,11 @@ class TestConfig:
         # terminal supports colors
         self.supports_colors = False
 
+        # Where to look up runtime stats produced by haddock, needed for
+        # the haddock perf tests in testsuite/tests/perf/haddock/.
+        # See Note [Haddock runtime stats files] at the bottom of this file.
+        self.stats_files_dir = '/please_set_stats_files_dir'
+
 global config
 config = TestConfig()
 
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index dc8b1b85f1..d0bd98015d 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -67,7 +67,6 @@ def isStatsTest():
     opts = getTestOpts()
     return opts.is_stats_test
 
-
 # This can be called at the top of a file of tests, to set default test options
 # for the following tests.
 def setTestOpts( f ):
@@ -1211,7 +1210,11 @@ def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
 
 def stats( name, way, stats_file ):
     opts = getTestOpts()
-    return check_stats(name, way, stats_file, opts.stats_range_fields)
+    return check_stats(name, way, in_testdir(stats_file), opts.stats_range_fields)
+
+def static_stats( name, way, stats_file ):
+    opts = getTestOpts()
+    return check_stats(name, way, in_statsdir(stats_file), opts.stats_range_fields)
 
 def metric_dict(name, way, metric, value):
     return Perf.PerfStat(
@@ -1234,7 +1237,7 @@ def check_stats(name, way, stats_file, range_fields):
     result = passed()
     if range_fields:
         try:
-            f = open(in_testdir(stats_file))
+            f = open(stats_file)
         except IOError as e:
             return failBecause(str(e))
         stats_file_contents = f.read()
@@ -1357,7 +1360,7 @@ def simple_build(name, way, extra_hc_opts, should_fail, top_mod, link, addsuf, b
     # ToDo: if the sub-shell was killed by ^C, then exit
 
     if isCompilerStatsTest():
-        statsResult = check_stats(name, way, stats_file, opts.stats_range_fields)
+        statsResult = check_stats(name, way, in_testdir(stats_file), opts.stats_range_fields)
         if badResult(statsResult):
             return statsResult
 
@@ -1442,7 +1445,7 @@ def simple_run(name, way, prog, extra_run_opts):
     if check_prof and not check_prof_ok(name, way):
         return failBecause('bad profile')
 
-    return check_stats(name, way, stats_file, opts.stats_range_fields)
+    return check_stats(name, way, in_testdir(stats_file), opts.stats_range_fields)
 
 def rts_flags(way):
     args = config.way_rts_flags.get(way, [])
@@ -2103,6 +2106,9 @@ def in_testdir(name, suffix=''):
 def in_srcdir(name, suffix=''):
     return os.path.join(getTestOpts().srcdir, add_suffix(name, suffix))
 
+def in_statsdir(name, suffix=''):
+    return os.path.join(config.stats_files_dir, add_suffix(name, suffix))
+
 # Finding the sample output.  The filename is of the form
 #
 #   <test>.stdout[-ws-<wordsize>][-<platform>|-<os>]