validate ansible-base's and collections runtime.yml (#69742)

* Validate ansible-base & collection's runtime.yml

Add new test `runtime-metadata`

* Schema validation of file
* Error if a a legacy meta/routing.yml exist in a collection
* removal_date OR removal_version

* Add tombstone validation.

* Allow both ISO 8601 date strings and datetime.date objects (from YAML dates).

* Address review comments.

* Add metadata to test collection.

* Add requirements file.

Co-authored-by: Felix Fontein <felix@fontein.de>
Co-authored-by: Matt Clay <matt@mystile.com>

6 files changed, 176 insertions, 0 deletions

diff --git a/changelogs/fragments/ansible-test-validate-runtime-file.yml b/changelogs/fragments/ansible-test-validate-runtime-file.yml
new file mode 100644
index 0000000000..6cff0872a2
--- /dev/null
+++ b/changelogs/fragments/ansible-test-validate-runtime-file.yml
@@ -0,0 +1,2 @@
+minor_changes:
+  - ansible-test now validates the schema of ansible_builtin_runtime.yml and a collections meta/runtime.yml file.
diff --git a/docs/docsite/rst/dev_guide/testing/sanity/runtime-metadata.rst b/docs/docsite/rst/dev_guide/testing/sanity/runtime-metadata.rst
new file mode 100644
index 0000000000..cf6d92724f
--- /dev/null
+++ b/docs/docsite/rst/dev_guide/testing/sanity/runtime-metadata.rst
@@ -0,0 +1,7 @@
+runtime-metadata.yml
+====================
+
+Validates the schema for:
+
+* ansible-base's ``lib/ansible/config/ansible_builtin_runtime.yml``
+* collection's ``meta/runtime.yml``
diff --git a/test/integration/targets/ansible-test/ansible_collections/ns/col/meta/runtime.yml b/test/integration/targets/ansible-test/ansible_collections/ns/col/meta/runtime.yml
new file mode 100644
index 0000000000..1ac15484db
--- /dev/null
+++ b/test/integration/targets/ansible-test/ansible_collections/ns/col/meta/runtime.yml
@@ -0,0 +1,4 @@
+plugin_routing:
+  modules:
+    hi:
+      redirect: hello
diff --git a/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt
new file mode 100644
index 0000000000..edd96991dd
--- /dev/null
+++ b/test/lib/ansible_test/_data/requirements/sanity.runtime-metadata.txt
@@ -0,0 +1,2 @@
+pyyaml
+voluptuous
diff --git a/test/lib/ansible_test/_data/sanity/code-smell/runtime-metadata.json b/test/lib/ansible_test/_data/sanity/code-smell/runtime-metadata.json
new file mode 100644
index 0000000000..44003ec0c9
--- /dev/null
+++ b/test/lib/ansible_test/_data/sanity/code-smell/runtime-metadata.json
@@ -0,0 +1,11 @@
+{
+    "prefixes": [
+        "lib/ansible/config/ansible_builtin_runtime.yml",
+        "meta/routing.yml",
+        "meta/runtime.yml"
+    ],
+    "extensions": [
+        ".yml"
+    ],
+    "output": "path-line-column-message"
+}
diff --git a/test/lib/ansible_test/_data/sanity/code-smell/runtime-metadata.py b/test/lib/ansible_test/_data/sanity/code-smell/runtime-metadata.py
new file mode 100755
index 0000000000..b986db2b57
--- /dev/null
+++ b/test/lib/ansible_test/_data/sanity/code-smell/runtime-metadata.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+"""Schema validation of ansible-base's ansible_builtin_runtime.yml and collection's meta/runtime.yml"""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import datetime
+import os
+import re
+import sys
+import yaml
+
+from voluptuous import Any, MultipleInvalid, PREVENT_EXTRA
+from voluptuous import Required, Schema, Invalid
+from voluptuous.humanize import humanize_error
+
+from ansible.module_utils.six import string_types
+
+
+def isodate(value):
+    """Validate a datetime.date or ISO 8601 date string."""
+    # datetime.date objects come from YAML dates, these are ok
+    if isinstance(value, datetime.date):
+        return value
+    # make sure we have a string
+    msg = 'Expected ISO 8601 date string (YYYY-MM-DD), or YAML date'
+    if not isinstance(value, string_types):
+        raise Invalid(msg)
+    try:
+        datetime.datetime.strptime(value, '%Y-%m-%d').date()
+    except ValueError:
+        raise Invalid(msg)
+    return value
+
+
+def validate_metadata_file(path):
+    """Validate explicit runtime metadata file"""
+    try:
+        with open(path, 'r') as f_path:
+            routing = yaml.safe_load(f_path)
+    except yaml.error.MarkedYAMLError as ex:
+        print('%s:%d:%d: YAML load failed: %s' % (path, ex.context_mark.line +
+                                                  1, ex.context_mark.column + 1, re.sub(r'\s+', ' ', str(ex))))
+        return
+    except Exception as ex:  # pylint: disable=broad-except
+        print('%s:%d:%d: YAML load failed: %s' %
+              (path, 0, 0, re.sub(r'\s+', ' ', str(ex))))
+        return
+
+    # Updates to schema MUST also be reflected in the documentation
+    # ~https://docs.ansible.com/ansible/devel/dev_guide/developing_collections.html
+
+    # plugin_routing schema
+
+    deprecation_tombstoning_schema = Any(Schema(
+        {
+            Required('removal_date'): Any(isodate),
+            'warning_text': Any(*string_types),
+        },
+        extra=PREVENT_EXTRA
+    ), Schema(
+        {
+            Required('removal_version'): Any(*string_types),
+            'warning_text': Any(*string_types),
+        },
+        extra=PREVENT_EXTRA
+    ))
+
+    plugin_routing_schema = Any(
+        Schema({
+            ('deprecation'): Any(deprecation_tombstoning_schema),
+            ('tombstone'): Any(deprecation_tombstoning_schema),
+            ('redirect'): Any(*string_types),
+        }, extra=PREVENT_EXTRA),
+    )
+
+    list_dict_plugin_routing_schema = [{str_type: plugin_routing_schema}
+                                       for str_type in string_types]
+
+    plugin_schema = Schema({
+        ('action'): Any(None, *list_dict_plugin_routing_schema),
+        ('become'): Any(None, *list_dict_plugin_routing_schema),
+        ('cache'): Any(None, *list_dict_plugin_routing_schema),
+        ('callback'): Any(None, *list_dict_plugin_routing_schema),
+        ('cliconf'): Any(None, *list_dict_plugin_routing_schema),
+        ('connection'): Any(None, *list_dict_plugin_routing_schema),
+        ('doc_fragments'): Any(None, *list_dict_plugin_routing_schema),
+        ('filter'): Any(None, *list_dict_plugin_routing_schema),
+        ('httpapi'): Any(None, *list_dict_plugin_routing_schema),
+        ('inventory'): Any(None, *list_dict_plugin_routing_schema),
+        ('lookup'): Any(None, *list_dict_plugin_routing_schema),
+        ('module_utils'): Any(None, *list_dict_plugin_routing_schema),
+        ('modules'): Any(None, *list_dict_plugin_routing_schema),
+        ('netconf'): Any(None, *list_dict_plugin_routing_schema),
+        ('shell'): Any(None, *list_dict_plugin_routing_schema),
+        ('strategy'): Any(None, *list_dict_plugin_routing_schema),
+        ('terminal'): Any(None, *list_dict_plugin_routing_schema),
+        ('test'): Any(None, *list_dict_plugin_routing_schema),
+        ('vars'): Any(None, *list_dict_plugin_routing_schema),
+    }, extra=PREVENT_EXTRA)
+
+    # import_redirection schema
+
+    import_redirection_schema = Any(
+        Schema({
+            ('redirect'): Any(*string_types),
+            # import_redirect doesn't currently support deprecation
+        }, extra=PREVENT_EXTRA)
+    )
+
+    list_dict_import_redirection_schema = [{str_type: import_redirection_schema}
+                                           for str_type in string_types]
+
+    # top level schema
+
+    schema = Schema({
+        # All of these are optional
+        ('plugin_routing'): Any(plugin_schema),
+        ('import_redirection'): Any(None, *list_dict_import_redirection_schema),
+        # requires_ansible: In the future we should validate this with SpecifierSet
+        ('requires_ansible'): Any(*string_types),
+        ('action_groups'): dict,
+    }, extra=PREVENT_EXTRA)
+
+    # Ensure schema is valid
+
+    try:
+        schema(routing)
+    except MultipleInvalid as ex:
+        for error in ex.errors:
+            # No way to get line/column numbers
+            print('%s:%d:%d: %s' % (path, 0, 0, humanize_error(routing, error)))
+
+
+def main():
+    """Validate runtime metadata"""
+    paths = sys.argv[1:] or sys.stdin.read().splitlines()
+
+    collection_legacy_file = 'meta/routing.yml'
+    collection_runtime_file = 'meta/runtime.yml'
+
+    for path in paths:
+        if path == collection_legacy_file:
+            print('%s:%d:%d: %s' % (path, 0, 0, ("Should be called '%s'" % collection_runtime_file)))
+            continue
+
+        validate_metadata_file(path)
+
+
+if __name__ == '__main__':
+    main()