CI: Always dump performance metrics.

7 files changed, 39 insertions, 21 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 01cf16cb6d..6d9f778419 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -279,7 +279,7 @@ hadrian-ghc-in-ghci:
       make binary-dist TAR_COMP_OPTS="-1"
     - |
       # Prepare to push git notes.
-      METRICS_FILE=$(mktemp)
+      METRICS_FILE=$CI_PROJECT_DIR/performance-metrics.tsv
       git config user.email "ben+ghc-ci@smart-cactus.org"
       git config user.name "GHC GitLab CI"
     - |
@@ -296,6 +296,7 @@ hadrian-ghc-in-ghci:
     paths:
       - ghc-*.tar.xz
       - junit.xml
+      - performance-metrics.tsv
 
 #################################
 # x86_64-darwin
diff --git a/testsuite/driver/my_typing.py b/testsuite/driver/my_typing.py
index d444f11af1..a31775d7f2 100644
--- a/testsuite/driver/my_typing.py
+++ b/testsuite/driver/my_typing.py
@@ -42,9 +42,4 @@ IssueNumber = NewType("IssueNumber", int)
 GitHash = NewType("GitHash", str)
 GitRef = NewType("GitRef", str)
 TestEnv = NewType("TestEnv", str)
-MetricName = NewType("MetricName", str)
-
-MetricBaselineOracle = Callable[[WayName, GitHash], Optional[float]]
-MetricDeviationOracle = Callable[[WayName, GitHash], Optional[float]]
-MetricOracles = NamedTuple("MetricOracles", [("baseline", MetricBaselineOracle),
-                                             ("deviation", MetricDeviationOracle)])
+MetricName = NewType("MetricName", str)
\ No newline at end of file
diff --git a/testsuite/driver/perf_notes.py b/testsuite/driver/perf_notes.py
index 78405895bf..cfbd5e529f 100644
--- a/testsuite/driver/perf_notes.py
+++ b/testsuite/driver/perf_notes.py
@@ -81,6 +81,11 @@ AllowedPerfChange = NamedTuple('AllowedPerfChange',
                                 ('opts', Dict[str, str])
                                 ])
 
+MetricBaselineOracle = Callable[[WayName, GitHash], Baseline]
+MetricDeviationOracle = Callable[[WayName, GitHash], Optional[float]]
+MetricOracles = NamedTuple("MetricOracles", [("baseline", MetricBaselineOracle),
+                                             ("deviation", MetricDeviationOracle)])
+
 def parse_perf_stat(stat_str: str) -> PerfStat:
     field_vals = stat_str.strip('\t').split('\t')
     return PerfStat(*field_vals) # type: ignore
@@ -183,7 +188,7 @@ def parse_allowed_perf_changes(commitMsg: str
 # Calculates a suggested string to append to the git commit in order to accept the
 # given changes.
 # changes: [(MetricChange, PerfStat)]
-def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat]]
+def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat, Any]]
                          ) -> str:
     Dec = MetricChange.Decrease
     Inc = MetricChange.Increase
@@ -193,7 +198,7 @@ def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat]]
 
     # Map tests to a map from change direction to metrics.
     test_to_dir_to_metrics = {} # type: Dict[TestName, Dict[MetricChange, List[MetricName]]]
-    for (change, perf_stat) in changes:
+    for (change, perf_stat, _baseline) in changes:
         change_dir_to_metrics = test_to_dir_to_metrics.setdefault(perf_stat.test, { Inc: [], Dec: [] })
         change_dir_to_metrics[change].append(perf_stat.metric)
 
@@ -246,12 +251,12 @@ def allow_changes_string(changes: List[Tuple[MetricChange, PerfStat]]
     return '\n\n'.join(msgs)
 
 # Formats a list of metrics into a string. Used e.g. to save metrics to a file or git note.
-def format_perf_stat(stats: Union[PerfStat, List[PerfStat]]) -> str:
+def format_perf_stat(stats: Union[PerfStat, List[PerfStat]], delimitor: str = "\t") -> str:
     # If a single stat, convert to a singleton list.
     if not isinstance(stats, list):
         stats = [stats]
 
-    return "\n".join(["\t".join([str(stat_val) for stat_val in stat]) for stat in stats])
+    return "\n".join([delimitor.join([str(stat_val) for stat_val in stat]) for stat in stats])
 
 # Appends a list of metrics to the git note of the given commit.
 # Tries up to max_tries times to write to git notes should it fail for some reason.
@@ -514,7 +519,8 @@ def get_commit_metric(gitNoteRef,
 # force_print: Print stats even if the test stat was in the tolerance range.
 # Returns a (MetricChange, pass/fail object) tuple. Passes if the stats are withing the expected value ranges.
 def check_stats_change(actual: PerfStat,
-                       baseline, tolerance_dev,
+                       baseline: Baseline,
+                       tolerance_dev,
                        allowed_perf_changes: Dict[TestName, List[AllowedPerfChange]] = {},
                        force_print = False
                        ) -> Tuple[MetricChange, Any]:
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index ebd433d3e7..5ef6ca27bf 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -25,7 +25,7 @@ import subprocess
 
 from testutil import getStdout, Watcher, str_warn, str_info
 from testglobals import getConfig, ghc_env, getTestRun, TestConfig, TestOptions, brokens
-from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty
+from perf_notes import MetricChange, inside_git_repo, is_worktree_dirty, format_perf_stat
 from junit import junit
 import cpu_features
 
@@ -392,6 +392,22 @@ else:
     # flush everything before we continue
     sys.stdout.flush()
 
+    # Dump metrics data.
+    if any(t.metrics):
+        print("\nPerformance Metrics:\n")
+        for (change, stat, baseline) in t.metrics:
+            if baseline is None:
+                print("{stat} [{direction}]".format(
+                    stat = format_perf_stat(stat),
+                    direction = str(change)))
+            else:
+                print("{stat} [{direction} from ({baselineStat}) @ HEAD~{depth}]".format(
+                    stat = format_perf_stat(stat),
+                    baselineStat = format_perf_stat(baseline.perfStat, ", "),
+                    direction = str(change),
+                    depth = baseline.commitDepth))
+        print("")
+
     # Warn if had to force skip perf tests (see Note force skip perf tests).
     spacing = "       "
     if forceSkipPerfTests and not args.skip_perf_tests:
@@ -401,7 +417,7 @@ else:
         print(spacing + 'You can still run the tests without git by specifying an output file with --metrics-file FILE.')
 
     # Warn of new metrics.
-    new_metrics = [metric for (change, metric) in t.metrics if change == MetricChange.NewMetric]
+    new_metrics = [metric for (change, metric, baseline) in t.metrics if change == MetricChange.NewMetric]
     if any(new_metrics):
         if inside_git_repo():
             reason = 'a baseline (expected value) cannot be recovered from' + \
@@ -441,7 +457,7 @@ else:
     summary(t, sys.stdout, config.no_print_summary, config.supports_colors)
 
     # Write perf stats if any exist or if a metrics file is specified.
-    stats = [stat for (_, stat) in t.metrics]
+    stats = [stat for (_, stat, __) in t.metrics]
     if hasMetricsFile:
         print('Appending ' + str(len(stats)) + ' stats to file: ' + config.metrics_file)
         with open(config.metrics_file, 'a') as f:
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 9895cd3a61..c39ca7a8c9 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -4,7 +4,7 @@
 
 from my_typing import *
 from pathlib import Path
-from perf_notes import MetricChange, PerfStat
+from perf_notes import MetricChange, PerfStat, Baseline, MetricOracles
 from datetime import datetime
 
 # -----------------------------------------------------------------------------
@@ -243,7 +243,7 @@ class TestRun:
        # [(change, PerfStat)] where change is one of the  MetricChange
        # constants: NewMetric, NoChange, Increase, Decrease.
        # NewMetric happens when the previous git commit has no metric recorded.
-       self.metrics = [] # type: List[Tuple[MetricChange, PerfStat]]
+       self.metrics = [] # type: List[Tuple[MetricChange, PerfStat, Optional[Baseline]]]
 
 global t
 t = TestRun()
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 136af73e1b..4177a2a15d 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -26,7 +26,7 @@ from testutil import strip_quotes, lndir, link_or_copy_file, passed, \
 import testutil
 from cpu_features import have_cpu_feature
 import perf_notes as Perf
-from perf_notes import MetricChange
+from perf_notes import MetricChange, PerfStat, MetricOracles
 extra_src_files = {'T4198': ['exitminus1.c']} # TODO: See #12223
 
 from my_typing import *
@@ -1317,7 +1317,7 @@ def static_stats( name, way, stats_file ):
     opts = getTestOpts()
     return check_stats(name, way, in_statsdir(stats_file), opts.stats_range_fields)
 
-def metric_dict(name, way, metric, value):
+def metric_dict(name, way, metric, value) -> PerfStat:
     return Perf.PerfStat(
         test_env = config.test_env,
         test     = name,
@@ -1380,7 +1380,7 @@ def check_stats(name: TestName,
                         tolerance_dev,
                         config.allowed_perf_changes,
                         config.verbose >= 4)
-                t.metrics.append((change, perf_stat))
+                t.metrics.append((change, perf_stat, baseline))
 
             # If any metric fails then the test fails.
             # Note, the remaining metrics are still run so that
diff --git a/testsuite/driver/testutil.py b/testsuite/driver/testutil.py
index 0c614f135a..071b641b7f 100644
--- a/testsuite/driver/testutil.py
+++ b/testsuite/driver/testutil.py
@@ -96,7 +96,7 @@ else:
         os.symlink(str(src), str(dst))
 
 class Watcher(object):
-    def __init__(self, count: int):
+    def __init__(self, count: int) -> None:
         self.pool = count
         self.evt = threading.Event()
         self.sync_lock = threading.Lock()