New feature flag functionality and fix includes failing silently (#367)

Build time feature flags are now defined in cloudinit/features.py.
Feature flags can be added to toggle configuration options or
deprecated features. Feature flag overrides can be placed in
cloudinit/feature_overrides.py. Further documentation can be found in
HACKING.rst.

Additionally, updated default behavior to exit with an exception
if #include can't retrieve resources as expected. This behavior
can be toggled with a feature flag.

LP: #1734939

1 files changed, 22 insertions, 9 deletions

diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 670dbee6..67bdf981 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -16,6 +16,7 @@ from email.mime.text import MIMEText
 
 from cloudinit import handlers
 from cloudinit import log as logging
+from cloudinit import features
 from cloudinit.url_helper import read_file_or_url, UrlError
 from cloudinit import util
 
@@ -69,6 +70,13 @@ def _set_filename(msg, filename):
                    'attachment', filename=str(filename))
 
 
+def _handle_error(error_message, source_exception=None):
+    if features.ERROR_ON_USER_DATA_FAILURE:
+        raise Exception(error_message) from source_exception
+    else:
+        LOG.warning(error_message)
+
+
 class UserDataProcessor(object):
     def __init__(self, paths):
         self.paths = paths
@@ -108,9 +116,11 @@ class UserDataProcessor(object):
                     ctype_orig = None
                     was_compressed = True
                 except util.DecompressionError as e:
-                    LOG.warning("Failed decompressing payload from %s of"
-                                " length %s due to: %s",
-                                ctype_orig, len(payload), e)
+                    error_message = (
+                        "Failed decompressing payload from {} of"
+                        " length {} due to: {}".format(
+                            ctype_orig, len(payload), e))
+                    _handle_error(error_message, e)
                     continue
 
             # Attempt to figure out the payloads content-type
@@ -231,19 +241,22 @@ class UserDataProcessor(object):
                     if resp.ok():
                         content = resp.contents
                     else:
-                        LOG.warning(("Fetching from %s resulted in"
-                                     " a invalid http code of %s"),
-                                    include_url, resp.code)
+                        error_message = (
+                            "Fetching from {} resulted in"
+                            " a invalid http code of {}".format(
+                                include_url, resp.code))
+                        _handle_error(error_message)
                 except UrlError as urle:
                     message = str(urle)
                     # Older versions of requests.exceptions.HTTPError may not
                     # include the errant url. Append it for clarity in logs.
                     if include_url not in message:
                         message += ' for url: {0}'.format(include_url)
-                    LOG.warning(message)
+                    _handle_error(message, urle)
                 except IOError as ioe:
-                    LOG.warning("Fetching from %s resulted in %s",
-                                include_url, ioe)
+                    error_message = "Fetching from {} resulted in {}".format(
+                        include_url, ioe)
+                    _handle_error(error_message, ioe)
 
             if content is not None:
                 new_msg = convert_string(content)