Add Strict Metaschema Validation (#1101)

Improve schema validation.

This adds strict validation of config module definitions at testing
time, with plumbing included for future runtime validation. This
eliminates a class of bugs resulting from schemas that have definitions
that are incorrect, but get interpreted by jsonschema as
"additionalProperties" that are therefore ignored.

- Add strict meta-schema for jsonschema unit test validation
- Separate schema from module metadata structure
- Improve type annotations for various functions and data types

Cleanup:
- Remove unused jsonschema "required" elements 
- Eliminate manual memoization in schema.py:get_schema(),
        reference module.__doc__ directly

1 files changed, 23 insertions, 1 deletions

diff --git a/cloudinit/importer.py b/cloudinit/importer.py
index f1194fbe..4e677af3 100644
--- a/cloudinit/importer.py
+++ b/cloudinit/importer.py
@@ -9,6 +9,27 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import sys
+import typing
+
+# annotations add value for development, but don't break old versions
+# pyver: 3.5 -> 3.8
+# pylint: disable=E1101
+if sys.version_info >= (3, 8) and hasattr(typing, "TypeDict"):
+    MetaSchema = typing.TypedDict(
+        "MetaSchema",
+        {
+            "name": str,
+            "id": str,
+            "title": str,
+            "description": str,
+            "distros": typing.List[str],
+            "examples": typing.List[str],
+            "frequency": str,
+        },
+    )
+else:
+    MetaSchema = dict
+# pylint: enable=E1101
 
 
 def import_module(module_name):
@@ -16,7 +37,8 @@ def import_module(module_name):
     return sys.modules[module_name]
 
 
-def find_module(base_name, search_paths, required_attrs=None):
+def find_module(base_name: str, search_paths, required_attrs=None) -> tuple:
+    """Finds and imports specified modules"""
     if not required_attrs:
         required_attrs = []
     # NOTE(harlowja): translate the search paths to include the base name.