util: Add a twister test tag validation script

While completing higher code coverage on herobrine, it has become
advantageous to create small hermetic tests to facilitate verifying
various enabled configs and faking of global functions. Twister test
tags are a way to group run many small tests together without them being
in the same directory.

Add twister_tags.py which contains a mapping of tag to description and
performs a presubmit check validating that all test tags in use have
been described.

BRANCH=none
BUG=b:249146559
TEST=repo upload with test commit
TEST=repo upload without test commit
TEST=util/twister_tags.py -l --validate-files

Signed-off-by: Aaron Massey <aaronmassey@google.com>
Change-Id: I3ec2aea239efe474b893d7d11e24bd4b10f3c0c6
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3919499
Reviewed-by: Jeremy Bettis <jbettis@chromium.org>
Reviewed-by: Keith Short <keithshort@chromium.org>

2 files changed, 91 insertions, 0 deletions

diff --git a/PRESUBMIT.cfg b/PRESUBMIT.cfg
index 3aabbe3bb3..c42cb7d9b9 100644
--- a/PRESUBMIT.cfg
+++ b/PRESUBMIT.cfg
@@ -32,3 +32,4 @@ config_option_check = util/config_option_check.py
 host_command_check = util/host_command_check.sh
 ec_commands_h = util/linux_ec_commands_h_check.sh
 migrated_files = util/migrated_files.sh ${PRESUBMIT_FILES}
+twister_test_tags = util/twister_tags.py --validate-files ${PRESUBMIT_FILES}
diff --git a/util/twister_tags.py b/util/twister_tags.py
new file mode 100755
index 0000000000..465ebab191
--- /dev/null
+++ b/util/twister_tags.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env vpython3
+
+# Copyright 2022 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+Script that contains definitions for Twister test tags while also checking
+testcase.yaml files to see if they only used predefined tags.
+"""
+
+# [VPYTHON:BEGIN]
+# python_version: "3.8"
+# wheel: <
+#   name: "infra/python/wheels/pyyaml-py3"
+#   version: "version:5.3.1"
+# >
+# [VPYTHON:END]
+
+import argparse
+import logging
+import os
+import sys
+
+import yaml  # pylint: disable=import-error
+
+TAG_TO_DESCRIPTION = {
+    "common": "Directly test shared code in the ec/common dir",
+    "system": "Directly test functions in common/system.c or shim/src/system.c",
+}
+
+SCRIPT_PATH = os.path.realpath(__file__)
+
+
+def main(args):
+    """List and/or validate testcase.yaml tags."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-l", "--list-tags", action="store_true")
+    parser.add_argument("--validate-files", nargs="+", type=str)
+    args = parser.parse_args()
+
+    list_tags = args.list_tags
+    validate_files = args.validate_files
+
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__file__)
+
+    if list_tags:
+        logger.info("Listing defined Twister testcase.yaml test tags:")
+        for tag, description in TAG_TO_DESCRIPTION.items():
+            logger.info("%s=%s", tag, description)
+
+    if validate_files:
+        testcase_yamls = []
+
+        bad_tag = False
+
+        for validated_file in validate_files:
+            if os.path.basename(validated_file) == "testcase.yaml":
+                testcase_yamls.append(validated_file)
+
+        def test_tags_generator(root):
+            """Returns space separated tags denoted by a tags key"""
+            if "tags" in root:
+                for tag in root["tags"].split(" "):
+                    yield tag
+
+            for val in root.values():
+                if isinstance(val, dict):
+                    for found in test_tags_generator(val):
+                        yield found
+
+        for testcase_yaml in testcase_yamls:
+            logger.info("Validating test tags in %s", testcase_yaml)
+            with open(testcase_yaml) as yaml_fd:
+                yaml_obj = yaml.safe_load(yaml_fd)
+                for tag in test_tags_generator(yaml_obj):
+                    if tag not in TAG_TO_DESCRIPTION:
+                        bad_tag = True
+                        # TODO(b/249280155) Suggest tag
+                        logger.error(
+                            f'Unrecognized tag: "{tag}" in {testcase_yaml}'
+                            + f' define "{tag}" in {SCRIPT_PATH}',
+                        )
+
+        if bad_tag:
+            sys.exit(1)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])