testsuite: Honour PERF_BASELINE_COMMIT when computing allowed metric changes

We now get all the commits between the PERF_BASELINE_COMMIT and HEAD and
check any of them for metric changes.

Fixes #20882

2 files changed, 21 insertions, 2 deletions

diff --git a/testsuite/driver/perf_notes.py b/testsuite/driver/perf_notes.py
index 1fd37bcb4e..3f67f0d57c 100644
--- a/testsuite/driver/perf_notes.py
+++ b/testsuite/driver/perf_notes.py
@@ -19,7 +19,7 @@ import subprocess
 import time
 import sys
 
-from collections import namedtuple
+from collections import namedtuple, defaultdict
 from math import ceil, trunc
 
 from testutil import passed, failBecause, testing_metrics, print_table
@@ -400,6 +400,24 @@ def commit_log(commitOrRange, n=None):
         print("Expected " + str(n) + " hashes, but git gave " + str(actualN) + ":\n" + output)
     return hashes
 
+def add_new_changes(changes, new_changes):
+   for key, new_change in new_changes.items():
+       changes[key].extend(new_change)
+
+def get_allowed_changes(baseline_ref: Optional[GitRef]) -> Dict[TestName, List[AllowedPerfChange]]:
+ if baseline_ref:
+        # The last 1000 commits in reverse order (starting from HEAD).
+        commit_hashes = baseline_commit_log(GitRef("HEAD"))
+        allowed_changes = defaultdict(list) # type: Dict[TestName, List[AllowedPerfChange]]
+        for commit in commit_hashes:
+            new_changes = get_allowed_perf_changes(commit)
+            if commit == baseline_ref: return dict(allowed_changes)
+            add_new_changes(allowed_changes, new_changes)
+        print("PERF_BASELINE_COMMIT not found in last 1000 commits...")
+        return dict()
+ else:
+        return get_allowed_perf_changes()
+
 # Cache of baseline values. This is a dict of dicts indexed on:
 # (useCiNamespace, commit) -> (test_env, test, metric, way) -> baseline
 # (bool          , str   ) -> (str     , str , str   , str) -> float
diff --git a/testsuite/driver/runtests.py b/testsuite/driver/runtests.py
index 219fb41001..6858715c7a 100644
--- a/testsuite/driver/runtests.py
+++ b/testsuite/driver/runtests.py
@@ -309,10 +309,11 @@ print('Compile ways: ' + ', '.join(config.compile_ways))
 
 # Try get allowed performance changes from the git commit.
 try:
-    config.allowed_perf_changes = Perf.get_allowed_perf_changes()
+    config.allowed_perf_changes = Perf.get_allowed_changes(config.baseline_commit)
 except subprocess.CalledProcessError:
     print('Failed to get allowed metric changes from the HEAD git commit message.')
 
+
 print('Allowing performance changes in: ' + ', '.join(config.allowed_perf_changes.keys()))
 
 # -----------------------------------------------------------------------------