Use cc_* module meta defintion over hardcoded vars (SC-888) (#1385)

Previously, each cc_* module required a hardcoded 'distro' and
'frequency' variable to control the respective distro to run on and
run frequency. If undefined or invalid, we silently changed this to
a default value.

Instead, this commit will validate the MetaSchema definition and raise
an exception if invalid. This means every module MUST contain a
MetaSchema definition.

Additionally, the Modules class was moved out of stages.py into its
own module, along with some helper functions in config/__init__.py.

1 files changed, 13 insertions, 226 deletions

diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 3dece336..5d0806a8 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -9,9 +9,9 @@ import os
 import pickle
 import sys
 from collections import namedtuple
-from typing import Dict, Set  # noqa: F401
+from typing import Dict, List, Optional, Set  # noqa: F401
 
-from cloudinit import cloud, config, distros, handlers, helpers, importer
+from cloudinit import cloud, distros, handlers, helpers, importer
 from cloudinit import log as logging
 from cloudinit import net, sources, type_utils, util
 from cloudinit.event import EventScope, EventType, userdata_to_events
@@ -29,7 +29,6 @@ from cloudinit.net import cmdline
 from cloudinit.reporting import events
 from cloudinit.settings import (
     CLOUD_CONFIG,
-    FREQUENCIES,
     PER_ALWAYS,
     PER_INSTANCE,
     PER_ONCE,
@@ -39,7 +38,6 @@ from cloudinit.sources import NetworkConfigSource
 
 LOG = logging.getLogger(__name__)
 
-NULL_DATA_SOURCE = None
 NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID"
 
 
@@ -98,17 +96,17 @@ def update_event_enabled(
 
 
 class Init(object):
-    def __init__(self, ds_deps=None, reporter=None):
+    def __init__(self, ds_deps: Optional[List[str]] = None, reporter=None):
         if ds_deps is not None:
             self.ds_deps = ds_deps
         else:
             self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
         # Created on first use
-        self._cfg = None
-        self._paths = None
-        self._distro = None
+        self._cfg: Optional[dict] = None
+        self._paths: Optional[helpers.Paths] = None
+        self._distro: Optional[distros.Distro] = None
         # Changed only when a fetch occurs
-        self.datasource = NULL_DATA_SOURCE
+        self.datasource: Optional[sources.DataSource] = None
         self.ds_restored = False
         self._previous_iid = None
 
@@ -126,7 +124,7 @@ class Init(object):
         self._paths = None
         self._distro = None
         if reset_ds:
-            self.datasource = NULL_DATA_SOURCE
+            self.datasource = None
             self.ds_restored = False
 
     @property
@@ -141,7 +139,7 @@ class Init(object):
             # If we have an active datasource we need to adjust
             # said datasource and move its distro/system config
             # from whatever it was to a new set...
-            if self.datasource is not NULL_DATA_SOURCE:
+            if self.datasource is not None:
                 self.datasource.distro = self._distro
                 self.datasource.sys_cfg = self.cfg
         return self._distro
@@ -252,7 +250,7 @@ class Init(object):
         return _pkl_load(self.paths.get_ipath_cur("obj_pkl"))
 
     def _write_to_cache(self):
-        if self.datasource is NULL_DATA_SOURCE:
+        if self.datasource is None:
             return False
         if util.get_cfg_option_bool(self.cfg, "manual_cache_clean", False):
             # The empty file in instance/ dir indicates manual cleaning,
@@ -301,7 +299,7 @@ class Init(object):
                 return (None, "cache invalid in datasource: %s" % ds)
 
     def _get_data_source(self, existing) -> sources.DataSource:
-        if self.datasource is not NULL_DATA_SOURCE:
+        if self.datasource is not None:
             return self.datasource
 
         with events.ReportEventStack(
@@ -330,7 +328,7 @@ class Init(object):
                 self.reporter,
             )
             LOG.info("Loaded datasource %s - %s", dsname, ds)
-        self.datasource = ds  # type: sources.DataSource
+        self.datasource = ds
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
         self._reset()
@@ -892,7 +890,7 @@ class Init(object):
             )
 
         if (
-            self.datasource is not NULL_DATA_SOURCE
+            self.datasource is not None
             and not self.is_new_instance()
             and not should_run_on_boot_event()
             and not event_enabled_and_metadata_updated(EventType.BOOT_LEGACY)
@@ -944,217 +942,6 @@ class Init(object):
             return
 
 
-class Modules(object):
-    def __init__(self, init, cfg_files=None, reporter=None):
-        self.init = init
-        self.cfg_files = cfg_files
-        # Created on first use
-        self._cached_cfg = None
-        if reporter is None:
-            reporter = events.ReportEventStack(
-                name="module-reporter",
-                description="module-desc",
-                reporting_enabled=False,
-            )
-        self.reporter = reporter
-
-    @property
-    def cfg(self):
-        # None check to avoid empty case causing re-reading
-        if self._cached_cfg is None:
-            merger = helpers.ConfigMerger(
-                paths=self.init.paths,
-                datasource=self.init.datasource,
-                additional_fns=self.cfg_files,
-                base_cfg=self.init.cfg,
-            )
-            self._cached_cfg = merger.cfg
-            # LOG.debug("Loading 'module' config %s", self._cached_cfg)
-        # Only give out a copy so that others can't modify this...
-        return copy.deepcopy(self._cached_cfg)
-
-    def _read_modules(self, name):
-        module_list = []
-        if name not in self.cfg:
-            return module_list
-        cfg_mods = self.cfg.get(name)
-        if not cfg_mods:
-            return module_list
-        # Create 'module_list', an array of hashes
-        # Where hash['mod'] = module name
-        #       hash['freq'] = frequency
-        #       hash['args'] = arguments
-        for item in cfg_mods:
-            if not item:
-                continue
-            if isinstance(item, str):
-                module_list.append(
-                    {
-                        "mod": item.strip(),
-                    }
-                )
-            elif isinstance(item, (list)):
-                contents = {}
-                # Meant to fall through...
-                if len(item) >= 1:
-                    contents["mod"] = item[0].strip()
-                if len(item) >= 2:
-                    contents["freq"] = item[1].strip()
-                if len(item) >= 3:
-                    contents["args"] = item[2:]
-                if contents:
-                    module_list.append(contents)
-            elif isinstance(item, (dict)):
-                contents = {}
-                valid = False
-                if "name" in item:
-                    contents["mod"] = item["name"].strip()
-                    valid = True
-                if "frequency" in item:
-                    contents["freq"] = item["frequency"].strip()
-                if "args" in item:
-                    contents["args"] = item["args"] or []
-                if contents and valid:
-                    module_list.append(contents)
-            else:
-                raise TypeError(
-                    "Failed to read '%s' item in config, unknown type %s"
-                    % (item, type_utils.obj_name(item))
-                )
-        return module_list
-
-    def _fixup_modules(self, raw_mods):
-        mostly_mods = []
-        for raw_mod in raw_mods:
-            raw_name = raw_mod["mod"]
-            freq = raw_mod.get("freq")
-            run_args = raw_mod.get("args") or []
-            mod_name = config.form_module_name(raw_name)
-            if not mod_name:
-                continue
-            if freq and freq not in FREQUENCIES:
-                LOG.warning(
-                    "Config specified module %s has an unknown frequency %s",
-                    raw_name,
-                    freq,
-                )
-                # Reset it so when ran it will get set to a known value
-                freq = None
-            mod_locs, looked_locs = importer.find_module(
-                mod_name, ["", type_utils.obj_name(config)], ["handle"]
-            )
-            if not mod_locs:
-                LOG.warning(
-                    "Could not find module named %s (searched %s)",
-                    mod_name,
-                    looked_locs,
-                )
-                continue
-            mod = config.fixup_module(importer.import_module(mod_locs[0]))
-            mostly_mods.append([mod, raw_name, freq, run_args])
-        return mostly_mods
-
-    def _run_modules(self, mostly_mods):
-        cc = self.init.cloudify()
-        # Return which ones ran
-        # and which ones failed + the exception of why it failed
-        failures = []
-        which_ran = []
-        for (mod, name, freq, args) in mostly_mods:
-            try:
-                # Try the modules frequency, otherwise fallback to a known one
-                if not freq:
-                    freq = mod.frequency
-                if freq not in FREQUENCIES:
-                    freq = PER_INSTANCE
-                LOG.debug(
-                    "Running module %s (%s) with frequency %s", name, mod, freq
-                )
-
-                # Use the configs logger and not our own
-                # TODO(harlowja): possibly check the module
-                # for having a LOG attr and just give it back
-                # its own logger?
-                func_args = [name, self.cfg, cc, config.LOG, args]
-                # Mark it as having started running
-                which_ran.append(name)
-                # This name will affect the semaphore name created
-                run_name = "config-%s" % (name)
-
-                desc = "running %s with frequency %s" % (run_name, freq)
-                myrep = events.ReportEventStack(
-                    name=run_name, description=desc, parent=self.reporter
-                )
-
-                with myrep:
-                    ran, _r = cc.run(
-                        run_name, mod.handle, func_args, freq=freq
-                    )
-                    if ran:
-                        myrep.message = "%s ran successfully" % run_name
-                    else:
-                        myrep.message = "%s previously ran" % run_name
-
-            except Exception as e:
-                util.logexc(LOG, "Running module %s (%s) failed", name, mod)
-                failures.append((name, e))
-        return (which_ran, failures)
-
-    def run_single(self, mod_name, args=None, freq=None):
-        # Form the users module 'specs'
-        mod_to_be = {
-            "mod": mod_name,
-            "args": args,
-            "freq": freq,
-        }
-        # Now resume doing the normal fixups and running
-        raw_mods = [mod_to_be]
-        mostly_mods = self._fixup_modules(raw_mods)
-        return self._run_modules(mostly_mods)
-
-    def run_section(self, section_name):
-        raw_mods = self._read_modules(section_name)
-        mostly_mods = self._fixup_modules(raw_mods)
-        d_name = self.init.distro.name
-
-        skipped = []
-        forced = []
-        overridden = self.cfg.get("unverified_modules", [])
-        active_mods = []
-        all_distros = set([distros.ALL_DISTROS])
-        for (mod, name, _freq, _args) in mostly_mods:
-            worked_distros = set(mod.distros)  # Minimally [] per fixup_modules
-            worked_distros.update(
-                distros.Distro.expand_osfamily(mod.osfamilies)
-            )
-
-            # Skip only when the following conditions are all met:
-            #  - distros are defined in the module != ALL_DISTROS
-            #  - the current d_name isn't in distros
-            #  - and the module is unverified and not in the unverified_modules
-            #    override list
-            if worked_distros and worked_distros != all_distros:
-                if d_name not in worked_distros:
-                    if name not in overridden:
-                        skipped.append(name)
-                        continue
-                    forced.append(name)
-            active_mods.append([mod, name, _freq, _args])
-
-        if skipped:
-            LOG.info(
-                "Skipping modules '%s' because they are not verified "
-                "on distro '%s'.  To run anyway, add them to "
-                "'unverified_modules' in config.",
-                ",".join(skipped),
-                d_name,
-            )
-        if forced:
-            LOG.info("running unverified_modules: '%s'", ", ".join(forced))
-
-        return self._run_modules(active_mods)
-
-
 def read_runtime_config():
     return util.read_conf(RUN_CLOUD_CONFIG)