tests/cachekey: Refactoring to now include an automatic updater

When cache key tests fail, and when it is intentional that
the keys have changed, then run ./tests/cachekey/update.py
to update the keys.

This is also useful whenever adding new tests.

2 files changed, 78 insertions, 15 deletions

diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 444da90e8..4f4ddc9e3 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -82,7 +82,7 @@ def parse_output_keys(output):
 # Returns an OrderedDict of element names
 # and their cache keys
 #
-def load_expected_keys(project_dir, actual_keys):
+def load_expected_keys(project_dir, actual_keys, raise_error=True):
 
     expected_keys = OrderedDict()
     for element_name in actual_keys:
@@ -92,11 +92,12 @@ def load_expected_keys(project_dir, actual_keys):
                 expected_key = f.read()
                 expected_key = expected_key.strip()
         except FileNotFoundError as e:
-            raise Exception("Cache key test needs update, " +
-                            "expected file {} not found.\n".format(expected) +
-                            "Hint: Actual key for element {} is: {}".format(
-                                element_name,
-                                actual_keys[element_name]))
+            expected_key = None
+            if raise_error:
+                raise Exception("Cache key test needs update, " +
+                                "expected file {} not found.\n\n".format(expected) +
+                                "Use tests/cachekey/update.py to automatically " +
+                                "update this test case")
 
         expected_keys[element_name] = expected_key
 
@@ -123,16 +124,9 @@ def assert_cache_keys(project_dir, output):
                     "    Expected: {}\n".format(expected_keys[element_name]) + \
                     "    Actual: {}\n".format(actual_keys[element_name])
 
-            # Write out the keys into files beside the project
-            # in the temp directory so that we can easily update
-            # the test when the artifact version changes.
-            filename = element_filename(project_dir, element_name, "actual")
-            with open(filename, "w") as f:
-                f.write(actual_keys[element_name])
-
         raise AssertionError("Cache key mismatches occurred:\n{}\n".format(info) +
-                             "New cache keys have been stored beside the " +
-                             "expected ones at: {}".format(project_dir))
+                             "Use tests/cachekey/update.py to automatically " +
+                             "update this test case")
 
 
 ##############################################
diff --git a/tests/cachekey/update.py b/tests/cachekey/update.py
new file mode 100755
index 000000000..09cf19657
--- /dev/null
+++ b/tests/cachekey/update.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Automatically create or update the .expected files in the
+# cache key test directory.
+#
+# Simply run without any arguments, from anywhere, e.g.:
+#
+#   ./tests/cachekey/update.py
+#
+# After this, add any files which were newly created and commit
+# the result in order to adjust the cache key test to changed
+# keys.
+#
+import os
+import tempfile
+from tests.testutils.runcli import Cli
+
+# This weird try / except is needed, because this will be imported differently
+# when pytest runner imports them vs when you run the updater directly from
+# this directory.
+try:
+    from cachekey import element_filename, parse_output_keys, load_expected_keys
+except ImportError:
+    from .cachekey import element_filename, parse_output_keys, load_expected_keys
+
+# Project directory
+PROJECT_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project",
+)
+
+
+def write_expected_key(element_name, actual_key):
+    expected_file = element_filename(PROJECT_DIR, element_name, 'expected')
+    with open(expected_file, 'w') as f:
+        f.write(actual_key)
+
+
+def update_keys():
+
+    with tempfile.TemporaryDirectory(dir=PROJECT_DIR) as tmpdir:
+        directory = os.path.join(tmpdir, 'cache')
+        os.makedirs(directory)
+        cli = Cli(directory, verbose=False)
+
+        # Run bst show
+        result = cli.run(project=PROJECT_DIR, silent=True, args=[
+            '--no-colors',
+            'show', '--format', '%{name}::%{full-key}',
+            'target.bst'
+        ])
+
+        # Load the actual keys, and the expected ones if they exist
+        actual_keys = parse_output_keys(result.output)
+        expected_keys = load_expected_keys(PROJECT_DIR, actual_keys, raise_error=False)
+
+        for element_name in actual_keys:
+            expected = element_filename(PROJECT_DIR, element_name, 'expected')
+
+            if actual_keys[element_name] != expected_keys[element_name]:
+                if not expected_keys[element_name]:
+                    print("Creating new expected file: {}".format(expected))
+                else:
+                    print("Updating expected file: {}".format(expected))
+
+                write_expected_key(element_name, actual_keys[element_name])
+
+if __name__ == '__main__':
+    update_keys()