testsuite: Make --ignore-perf-tests more expressive

Allow skipping of only increases/decreases.

4 files changed, 22 insertions, 9 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b8004f149b..4af4247481 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -496,7 +496,7 @@ validate-x86_64-darwin:
     TEST_ENV: "x86_64-darwin"
     BUILD_FLAVOUR: "validate"
     # Due to #19025
-    RUNTEST_ARGS: "--ignore-perf-failures"
+    RUNTEST_ARGS: "--ignore-perf-failures=all"
   after_script:
     - cp -Rf $HOME/.cabal cabal-cache
     - .gitlab/ci.sh clean
@@ -523,7 +523,7 @@ validate-x86_64-darwin:
     CONFIGURE_ARGS: --with-intree-gmp
     TEST_ENV: "x86_64-darwin-hadrian"
     # Due to #19025
-    RUNTEST_ARGS: "--ignore-perf-failures"
+    RUNTEST_ARGS: "--ignore-perf-failures=all"
     BIN_DIST_NAME: "ghc-x86_64-apple-darwin"
     BUILD_FLAVOUR: "validate"
   script:
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 577851cf46..d4425b5da5 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -76,7 +76,8 @@ parser.add_argument("--perf-baseline", type=GitRef, metavar='COMMIT', help="Base
 parser.add_argument("--test-package-db", dest="test_package_db", action="append", help="Package db providing optional packages used by the testsuite.")
 perf_group.add_argument("--skip-perf-tests", action="store_true", help="skip performance tests")
 perf_group.add_argument("--only-perf-tests", action="store_true", help="Only do performance tests")
-perf_group.add_argument("--ignore-perf-failures", action="store_true", help="Don't fail due to out-of-tolerance perf tests")
+perf_group.add_argument("--ignore-perf-failures", choices=['increases','decreases','all'],
+                        help="Do not fail due to out-of-tolerance perf tests")
 
 args = parser.parse_args()
 
@@ -152,7 +153,13 @@ if args.verbose is not None:
 forceSkipPerfTests = not hasMetricsFile and not inside_git_repo()
 config.skip_perf_tests = args.skip_perf_tests or forceSkipPerfTests
 config.only_perf_tests = args.only_perf_tests
-config.ignore_perf_failures = args.ignore_perf_failures
+if args.ignore_perf_failures == 'all':
+    config.ignore_perf_decreases = True
+    config.ignore_perf_increases = True
+elif args.ignore_perf_failures == 'increases':
+    config.ignore_perf_increases = True
+elif args.ignore_perf_failures == 'decreases':
+    config.ignore_perf_decreases = True
 
 if args.test_env:
     config.test_env = args.test_env
diff --git a/testsuite/driver/testglobals.py b/testsuite/driver/testglobals.py
index 117df41eb6..c7140387af 100644
--- a/testsuite/driver/testglobals.py
+++ b/testsuite/driver/testglobals.py
@@ -32,7 +32,8 @@ class TestConfig:
         self.only = set()
 
         # Don't fail on out-of-tolerance stat failures
-        self.ignore_perf_failures = False
+        self.ignore_perf_increases = False
+        self.ignore_perf_decreases = False
 
         # Accept new output which differs from the sample?
         self.accept = False
diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py
index 2375ea8c20..4ccc790e37 100644
--- a/testsuite/driver/testlib.py
+++ b/testsuite/driver/testlib.py
@@ -1506,6 +1506,7 @@ def check_stats(name: TestName,
         for (metric, baseline_and_dev) in range_fields.items():
             # Remove any metric prefix e.g. "runtime/" and "compile_time/"
             stat_file_metric = metric.split("/")[-1]
+            perf_change = None
 
             field_match = re.search('\\("' + stat_file_metric + '", "([0-9]+)"\\)', stats_file_contents)
             if field_match is None:
@@ -1518,14 +1519,13 @@ def check_stats(name: TestName,
 
                 # Store the metric so it can later be stored in a git note.
                 perf_stat = metric_dict(name, way, metric, actual_val)
-                change = None
 
                 # If this is the first time running the benchmark, then pass.
                 baseline = baseline_and_dev.baseline(way, head_commit) \
                     if Perf.inside_git_repo() else None
                 if baseline is None:
                     metric_result = passed()
-                    change = MetricChange.NewMetric
+                    perf_change = MetricChange.NewMetric
                 else:
                     tolerance_dev = baseline_and_dev.deviation
                     (change, metric_result) = Perf.check_stats_change(
@@ -1534,12 +1534,17 @@ def check_stats(name: TestName,
                         tolerance_dev,
                         config.allowed_perf_changes,
                         config.verbose >= 4)
-                t.metrics.append(PerfMetric(change=change, stat=perf_stat, baseline=baseline))
+                t.metrics.append(PerfMetric(change=perf_change, stat=perf_stat, baseline=baseline))
 
             # If any metric fails then the test fails.
             # Note, the remaining metrics are still run so that
             # a complete list of changes can be presented to the user.
-            if metric_result.passFail == 'fail' and not config.ignore_perf_failures:
+            if metric_result.passFail == 'fail':
+                if config.ignore_perf_increases and change == MetricChange.Increase:
+                    pass
+                elif config.ignore_perf_decreases and change == MetricChange.Decrease:
+                    pass
+                else:
                 result = metric_result
 
     return result