Initial hotplug support

Adds a udev script which will invoke a hotplug hook script on all net
add events. The script will write some udev arguments to a systemd FIFO
socket (to ensure we have only instance of cloud-init running at a
time), which is then read by a new service that calls a new 'cloud-init
devel hotplug-hook' command to handle the new event.

This hotplug-hook command will:
- Fetch the pickled datsource
- Verify that the hotplug event is supported/enabled
- Update the metadata for the datasource
- Ensure the hotplugged device exists within the datasource
- Apply the config change on the datasource metadata
- Bring up the new interface (or apply global network configuration)
- Save the updated metadata back to the pickle cache

Also scattered in some unrelated typing to make my life easier.

14 files changed, 318 insertions, 11 deletions

diff --git a/bash_completion/cloud-init b/bash_completion/cloud-init
index a9577e9d..9043bb51 100644
--- a/bash_completion/cloud-init
+++ b/bash_completion/cloud-init
@@ -28,7 +28,7 @@ _cloudinit_complete()
                     COMPREPLY=($(compgen -W "--help --tarfile --include-userdata" -- $cur_word))
                     ;;
                 devel)
-                    COMPREPLY=($(compgen -W "--help schema net-convert" -- $cur_word))
+                    COMPREPLY=($(compgen -W "--help hotplug-book schema net-convert" -- $cur_word))
                     ;;
                 dhclient-hook)
                     COMPREPLY=($(compgen -W "--help up down" -- $cur_word))
@@ -64,6 +64,9 @@ _cloudinit_complete()
                 --frequency)
                     COMPREPLY=($(compgen -W "--help instance always once" -- $cur_word))
                     ;;
+                hotplug-hook)
+                    COMPREPLY=($(compgen -W "--help" -- $cur_word))
+                    ;;
                 net-convert)
                     COMPREPLY=($(compgen -W "--help --network-data --kind --directory --output-kind" -- $cur_word))
                     ;;
diff --git a/cloudinit/cmd/devel/hotplug_hook.py b/cloudinit/cmd/devel/hotplug_hook.py
new file mode 100644
index 00000000..16a3df6b
--- /dev/null
+++ b/cloudinit/cmd/devel/hotplug_hook.py
@@ -0,0 +1,220 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Handle reconfiguration on hotplug events"""
+import abc
+import argparse
+import os
+import six
+import time
+
+from cloudinit import log
+from cloudinit import reporting
+from cloudinit.event import EventScope, EventType
+from cloudinit.net import read_sys_net_safe
+from cloudinit.net.network_state import parse_net_config_data
+from cloudinit.reporting import events
+from cloudinit.stages import Init
+from cloudinit.sources import DataSource
+
+
+LOG = log.getLogger(__name__)
+NAME = 'hotplug-hook'
+
+
+def get_parser(parser=None):
+    """Build or extend and arg parser for hotplug-hook utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        subcommand which will be extended to support the args of this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
+
+    parser.add_argument("-d", "--devpath",
+                        metavar="PATH",
+                        help="sysfs path to hotplugged device")
+    parser.add_argument("-s", "--subsystem",
+                        choices=['net', 'block'])
+    parser.add_argument("-u", "--udevaction",
+                        choices=['add', 'change', 'remove'])
+
+    return parser
+
+
+@six.add_metaclass(abc.ABCMeta)
+class UeventHandler(object):
+    def __init__(self, id, datasource, devpath, success_fn):
+        self.id = id
+        self.datasource = datasource  # type: DataSource
+        self.devpath = devpath
+        self.success_fn = success_fn
+
+    @abc.abstractmethod
+    def apply(self):
+        raise NotImplementedError()
+
+    @property
+    @abc.abstractmethod
+    def config(self):
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def device_detected(self) -> bool:
+        raise NotImplementedError()
+
+    def detect_hotplugged_device(self, action):
+        detect_presence = None
+        if action == 'add':
+            detect_presence = True
+        elif action == 'remove':
+            detect_presence = False
+        else:
+            raise ValueError('Cannot detect unknown action: %s' % action)
+
+        if detect_presence != self.device_detected():
+            raise RuntimeError(
+                'Failed to detect %s in updated metadata' % self.id)
+
+    def success(self):
+        return self.success_fn()
+
+    def update_metadata(self):
+        result = self.datasource.update_metadata_if_supported([
+            EventType.HOTPLUG])
+        if not result:
+            raise RuntimeError(
+                'Datasource %s not updated for '
+                'event %s' % (self.datasource, EventType.HOTPLUG)
+            )
+        return result
+
+
+class NetHandler(UeventHandler):
+    def __init__(self, datasource, devpath, success_fn):
+        # convert devpath to mac address
+        id = read_sys_net_safe(os.path.basename(devpath), 'address')
+        super().__init__(id, datasource, devpath, success_fn)
+
+    def apply(self):
+        if self.datasource.distro.apply_network_config(
+            self.config,
+            bring_up=True,
+            devices=[os.path.basename(self.devpath)],
+        ):
+            # apply_network_config returns **False** on success
+            raise RuntimeError(
+                'Failed to bring devices: {}'.format(self.devpath))
+
+    @property
+    def config(self):
+        return self.datasource.network_config
+
+    def device_detected(self) -> bool:
+        netstate = parse_net_config_data(self.config)
+        found = [
+            iface for iface in netstate.iter_interfaces()
+            if iface.get('mac_address') == self.id
+        ]
+        LOG.debug('Ifaces with ID=%s : %s' % (self.id, found))
+        return len(found) > 0
+
+
+SUBSYSTEM_PROPERTES_MAP = {
+    'net': (NetHandler, EventScope.NETWORK),
+}
+
+
+def handle_hotplug(
+    hotplug_init: Init, devpath, subsystem, udevaction
+):
+    handler_cls, event_scope = SUBSYSTEM_PROPERTES_MAP.get(
+        subsystem, (None, None)
+    )
+    if handler_cls is None:
+        raise Exception(
+            'hotplug-hook: cannot handle events for subsystem: {}'.format(
+                subsystem))
+
+    LOG.debug('Fetching datasource')
+    datasource = hotplug_init.fetch(existing="trust")
+
+    if not hotplug_init.update_event_enabled(
+        event_source_type=EventType.HOTPLUG,
+        scope=EventScope.NETWORK
+    ):
+        LOG.debug('hotplug not enabled for event of type %s', event_scope)
+        return
+
+    LOG.debug('Creating %s event handler' % subsystem)
+    event_handler = handler_cls(
+        datasource=datasource,
+        devpath=devpath,
+        success_fn=hotplug_init._write_to_cache
+    )  # type: UeventHandler
+    wait_times = [1, 1, 1, 3, 5]
+    for attempt, wait in enumerate(wait_times):
+        LOG.debug(
+            'subsystem=%s update attempt %s/%s' % (
+                subsystem,
+                attempt,
+                len(wait_times)))
+        try:
+            LOG.debug('Refreshing metadata')
+            event_handler.update_metadata()
+            LOG.debug('Detecting device in updated metadata')
+            event_handler.detect_hotplugged_device(action=udevaction)
+            LOG.debug('Applying config change')
+            event_handler.apply()
+            LOG.debug('Updating cache')
+            event_handler.success()
+            break
+        except Exception as e:
+            if attempt + 1 >= len(wait_times):
+                raise
+            LOG.debug('Exception while processing hotplug event. %s' % e)
+            time.sleep(wait)
+
+
+def handle_args(name, args):
+    # Note that if an exception happens between now and when logging is
+    # setup, we'll only see it in the journal
+    hotplug_reporter = events.ReportEventStack(
+        name, __doc__, reporting_enabled=True
+    )
+
+    hotplug_init = Init(ds_deps=[], reporter=hotplug_reporter)
+    hotplug_init.read_cfg()
+
+    log.setupLogging(hotplug_init.cfg)
+    if 'reporting' in hotplug_init.cfg:
+        reporting.update_configuration(hotplug_init.cfg.get('reporting'))
+
+    # Logging isn't be setup until now
+    LOG.debug(
+        '%s called with the following arguments:\n'
+        'udevaction: %s\n'
+        'subsystem: %s\n'
+        'devpath: %s',
+        name, args.udevaction, args.subsystem, args.devpath
+    )
+
+    with hotplug_reporter:
+        try:
+            handle_hotplug(
+                hotplug_init=hotplug_init,
+                devpath=args.devpath,
+                subsystem=args.subsystem,
+                udevaction=args.udevaction,
+            )
+        except Exception:
+            LOG.exception('Received fatal exception handling hotplug!')
+            raise
+
+    LOG.debug('Exiting hotplug handler')
+    reporting.flush_events()
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
+    handle_args(NAME, args)
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
index 1a3c46a4..be304630 100644
--- a/cloudinit/cmd/devel/parser.py
+++ b/cloudinit/cmd/devel/parser.py
@@ -7,6 +7,7 @@
 import argparse
 from cloudinit.config import schema
 
+from . import hotplug_hook
 from . import net_convert
 from . import render
 from . import make_mime
@@ -21,6 +22,8 @@ def get_parser(parser=None):
     subparsers.required = True
 
     subcmds = [
+        (hotplug_hook.NAME, hotplug_hook.__doc__,
+         hotplug_hook.get_parser, hotplug_hook.handle_args),
         ('schema', 'Validate cloud-config files for document schema',
          schema.get_parser, schema.handle_schema_args),
         (net_convert.NAME, net_convert.__doc__,
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 2caa8bc2..39736176 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -14,6 +14,7 @@ import os
 import re
 import stat
 import string
+from typing import Iterable
 import urllib.parse
 from io import StringIO
 from typing import Any, Mapping
@@ -206,8 +207,17 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
     def generate_fallback_config(self):
         return net.generate_fallback_config()
 
-    def apply_network_config(self, netconfig, bring_up=False):
-        # apply network config netconfig
+    def apply_network_config(
+        self, netconfig, bring_up=False, devices: Iterable[str] = None
+    ) -> bool:
+        """Apply the network config.
+
+        If bring_up is True, attempt to bring up the passed in devices. If
+        devices is None, attempt to bring up devices returned by
+        _write_network_config.
+
+        Returns True if any devices failed to come up, otherwise False.
+        """
         # This method is preferred to apply_network which only takes
         # a much less complete network config format (interfaces(5)).
         network_state = parse_net_config_data(netconfig)
@@ -221,7 +231,10 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
         # Now try to bring them up
         if bring_up:
             network_activator = activators.select_activator()
-            network_activator.bring_up_all_interfaces(network_state)
+            if devices:
+                network_activator.bring_up_interfaces(devices)
+            else:
+                network_activator.bring_up_all_interfaces(network_state)
         return False
 
     def apply_network_config_names(self, netconfig):
diff --git a/cloudinit/event.py b/cloudinit/event.py
index 76a0afc6..53ad4c25 100644
--- a/cloudinit/event.py
+++ b/cloudinit/event.py
@@ -29,6 +29,7 @@ class EventType(Enum):
     BOOT = "boot"
     BOOT_NEW_INSTANCE = "boot-new-instance"
     BOOT_LEGACY = "boot-legacy"
+    HOTPLUG = 'hotplug'
 
     def __str__(self):  # pylint: disable=invalid-str-returned
         return self.value
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 62756cf7..19c8d126 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -12,9 +12,8 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import subp
 from cloudinit import util
-
+from cloudinit.event import EventScope, EventType
 from cloudinit.net import eni
-
 from cloudinit.sources.DataSourceIBMCloud import get_ibm_platform
 from cloudinit.sources.helpers import openstack
 
@@ -37,6 +36,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
 
     dsname = 'ConfigDrive'
 
+    supported_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+        EventType.HOTPLUG,
+    }}
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths)
         self.source = None
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 619a171e..a85b71d7 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -8,11 +8,11 @@ import time
 
 from cloudinit import dmi
 from cloudinit import log as logging
-from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
-
+from cloudinit.event import EventScope, EventType
+from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
 from cloudinit.sources.helpers import openstack
 from cloudinit.sources import DataSourceOracle as oracle
 
@@ -46,6 +46,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
     # Whether we want to get network configuration from the metadata service.
     perform_dhcp_setup = False
 
+    supported_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+        EventType.HOTPLUG
+    }}
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
         self.metadata_address = None
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index a07c4b4f..436cd9b7 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -23,6 +23,7 @@ from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
 from cloudinit.atomic_helper import write_json
+from cloudinit.distros import Distro
 from cloudinit.event import EventScope, EventType
 from cloudinit.filters import launch_index
 from cloudinit.persistence import CloudInitPickleMixin
@@ -211,7 +212,7 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
 
     _ci_pkl_version = 1
 
-    def __init__(self, sys_cfg, distro, paths, ud_proc=None):
+    def __init__(self, sys_cfg, distro: Distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
         self.paths = paths
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 3688be2e..e02a2f4c 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -241,7 +241,7 @@ class Init(object):
             else:
                 return (None, "cache invalid in datasource: %s" % ds)
 
-    def _get_data_source(self, existing):
+    def _get_data_source(self, existing) -> sources.DataSource:
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
@@ -267,7 +267,7 @@ class Init(object):
                                                cfg_list,
                                                pkg_list, self.reporter)
             LOG.info("Loaded datasource %s - %s", dsname, ds)
-        self.datasource = ds
+        self.datasource = ds  # type: sources.DataSource
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
         self._reset()
diff --git a/setup.py b/setup.py
index dcbe0843..7fa03e63 100755
--- a/setup.py
+++ b/setup.py
@@ -128,6 +128,7 @@ INITSYS_FILES = {
     'systemd': [render_tmpl(f)
                 for f in (glob('systemd/*.tmpl') +
                           glob('systemd/*.service') +
+                          glob('systemd/*.socket') +
                           glob('systemd/*.target'))
                 if (is_f(f) and not is_generator(f))],
     'systemd.generators': [
@@ -249,6 +250,7 @@ data_files = [
     (ETC + '/cloud/cloud.cfg.d', glob('config/cloud.cfg.d/*')),
     (ETC + '/cloud/templates', glob('templates/*')),
     (USR_LIB_EXEC + '/cloud-init', ['tools/ds-identify',
+                                    'tools/hook-hotplug',
                                     'tools/uncloud-init',
                                     'tools/write-ssh-key-fingerprints']),
     (USR + '/share/bash-completion/completions',
diff --git a/systemd/cloud-init-hotplugd.service b/systemd/cloud-init-hotplugd.service
new file mode 100755
index 00000000..6f231cdc
--- /dev/null
+++ b/systemd/cloud-init-hotplugd.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=cloud-init hotplug hook daemon
+After=cloud-init-hotplugd.socket
+
+[Service]
+Type=simple
+ExecStart=/bin/bash -c 'read args <&3; echo "args=$args"; \
+                        exec /usr/bin/cloud-init devel hotplug-hook $args; \
+                        exit 0'
+SyslogIdentifier=cloud-init-hotplugd
+TimeoutStopSec=5
diff --git a/systemd/cloud-init-hotplugd.socket b/systemd/cloud-init-hotplugd.socket
new file mode 100644
index 00000000..f8f10486
--- /dev/null
+++ b/systemd/cloud-init-hotplugd.socket
@@ -0,0 +1,8 @@
+[Unit]
+Description=cloud-init hotplug hook socket
+
+[Socket]
+ListenFIFO=/run/cloud-init/hook-hotplug-cmd
+
+[Install]
+WantedBy=cloud-init.target
diff --git a/tools/hook-hotplug b/tools/hook-hotplug
new file mode 100755
index 00000000..cd8af627
--- /dev/null
+++ b/tools/hook-hotplug
@@ -0,0 +1,26 @@
+#!/bin/bash
+# This file is part of cloud-init. See LICENSE file for license information.
+
+# This script checks if cloud-init has hotplug hooked and if
+# cloud-init has finished; if so invoke cloud-init hotplug-hook
+
+is_finished() {
+    [ -e /run/cloud-init/result.json ]
+}
+
+if is_finished; then
+    # only hook pci devices at this time
+    case "${DEVPATH}" in
+        /devices/pci*)
+            # open cloud-init's hotplug-hook fifo rw
+            exec 3<>/run/cloud-init/hook-hotplug-cmd
+            env_params=(
+                --devpath="${DEVPATH}"
+                --subsystem="${SUBSYSTEM}"
+                --udevaction="${ACTION}"
+            )
+            # write params to cloud-init's hotplug-hook fifo
+            echo "${env_params[@]}" >&3
+            ;;
+    esac
+fi
diff --git a/udev/10-cloud-init-hook-hotplug.rules b/udev/10-cloud-init-hook-hotplug.rules
new file mode 100644
index 00000000..648e7bfc
--- /dev/null
+++ b/udev/10-cloud-init-hook-hotplug.rules
@@ -0,0 +1,6 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+# Handle device adds only
+ACTION!="add", GOTO="cloudinit_end"
+LABEL="cloudinit_hook"
+SUBSYSTEM=="net|block", RUN+="/usr/lib/cloud-init/hook-hotplug"
+LABEL="cloudinit_end"