20.4-66-gad23441d-0ubuntu1 (patches unapplied)

Imported using git-ubuntu import.

21 files changed, 641 insertions, 124 deletions

diff --git a/.travis.yml b/.travis.yml
index 8ead3468..f15752bc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -41,86 +41,6 @@ matrix:
     fast_finish: true
     include:
         - python: 3.6
-        - if: NOT branch =~ /^ubuntu\//
-          env: {}
-          cache:
-              - directories:
-                  - lxd_images
-                  - chroots
-          before_cache:
-              - |
-                  # Find the most recent image file
-                  latest_file="$(sudo ls -Art /var/snap/lxd/common/lxd/images/ | tail -n 1)"
-                  # This might be <hash>.rootfs or <hash>, normalise
-                  latest_file="$(basename $latest_file .rootfs)"
-                  # Find all files with that prefix and copy them to our cache dir
-                  sudo find /var/snap/lxd/common/lxd/images/ -name $latest_file* -print -exec cp {} "$TRAVIS_BUILD_DIR/lxd_images/" \;
-          install:
-            - git fetch --unshallow
-            - sudo apt-get install -y --install-recommends sbuild ubuntu-dev-tools fakeroot tox debhelper
-            - pip install .
-            - pip install tox
-            # bionic has lxd from deb installed, remove it first to ensure
-            # pylxd talks only to the lxd from snap
-            - sudo apt remove --purge lxd lxd-client
-            - sudo rm -Rf /var/lib/lxd
-            - sudo snap install lxd
-            - sudo lxd init --auto
-            - sudo mkdir --mode=1777 -p /var/snap/lxd/common/consoles
-            # Move any cached lxd images into lxd's image dir
-            - sudo find "$TRAVIS_BUILD_DIR/lxd_images/" -type f -print -exec mv {} /var/snap/lxd/common/lxd/images/ \;
-            - sudo usermod -a -G lxd $USER
-            - sudo sbuild-adduser $USER
-            - cp /usr/share/doc/sbuild/examples/example.sbuildrc /home/$USER/.sbuildrc
-          script:
-            # Ubuntu LTS: Build
-            - ./packages/bddeb -S -d --release xenial
-            - |
-                needs_caching=false
-                if [ -e "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" ]; then
-                    # If we have a cached chroot, move it into place
-                    sudo mkdir -p /var/lib/schroot/chroots/xenial-amd64
-                    sudo tar --sparse --xattrs --preserve-permissions --numeric-owner -xf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64
-                    # Write its configuration
-                    cat > sbuild-xenial-amd64 << EOM
-                [xenial-amd64]
-                description=xenial-amd64
-                groups=sbuild,root,admin
-                root-groups=sbuild,root,admin
-                # Uncomment these lines to allow members of these groups to access
-                # the -source chroots directly (useful for automated updates, etc).
-                #source-root-users=sbuild,root,admin
-                #source-root-groups=sbuild,root,admin
-                type=directory
-                profile=sbuild
-                union-type=overlay
-                directory=/var/lib/schroot/chroots/xenial-amd64
-                EOM
-                    sudo mv sbuild-xenial-amd64 /etc/schroot/chroot.d/
-                    sudo chown root /etc/schroot/chroot.d/sbuild-xenial-amd64
-                    # And ensure it's up-to-date.
-                    before_pkgs="$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)"
-                    sudo schroot -c source:xenial-amd64 -d / -- sh -c "apt-get update && apt-get -qqy upgrade"
-                    after_pkgs=$(sudo schroot -c source:xenial-amd64 -d / dpkg -l | sha256sum)
-                    if [ "$before_pkgs" != "$after_pkgs" ]; then
-                        needs_caching=true
-                    fi
-                else
-                    # Otherwise, create the chroot
-                    sudo -E su $USER -c 'mk-sbuild xenial'
-                    needs_caching=true
-                fi
-                # If there are changes to the schroot (or it's entirely new),
-                # tar up the schroot (to preserve ownership/permissions) and
-                # move it into the cached dir; no need to compress it because
-                # Travis will do that anyway
-                if [ "$needs_caching" = "true" ]; then
-                    sudo tar --sparse --xattrs --xattrs-include=* -cf "$TRAVIS_BUILD_DIR/chroots/xenial-amd64.tar" -C /var/lib/schroot/chroots/xenial-amd64 .
-                fi
-            # Use sudo to get a new shell where we're in the sbuild group
-            - sudo -E su $USER -c 'sbuild --nolog --no-run-lintian --verbose --dist=xenial cloud-init_*.dsc'
-            # Ubuntu LTS: Integration
-            - sg lxd -c 'tox -e citest -- run --verbose --preserve-data --data-dir results --os-name xenial --test modules/apt_configure_sources_list.yaml --test modules/ntp_servers --test modules/set_password_list --test modules/user_groups --deb cloud-init_*_all.deb'
         - name: "Integration Tests"
           if: NOT branch =~ /^ubuntu\//
           env: {}
diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 4fb9b44e..911789c7 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -24,15 +24,19 @@ Configuration for this module is under the ``random_seed`` config key. The
 optionally be specified in encoded form, with the encoding specified in
 ``encoding``.
 
+If the cloud provides its own random seed data, it will be appended to ``data``
+before it is written to ``file``.
+
 .. note::
     when using a multiline value for ``data`` or specifying binary data, be
     sure to follow yaml syntax and use the ``|`` and ``!binary`` yaml format
     specifiers when appropriate
 
-Instead of specifying a data string, a command can be run to generate/collect
-the data to be written. The command should be specified as a list of args in
-the ``command`` key. If a command is specified that cannot be run, no error
-will be reported unless ``command_required`` is set to true.
+If the ``command`` key is specified, the given command will be executed.  This
+will happen after ``file`` has been populated.  That command's environment will
+contain the value of the ``file`` key as ``RANDOM_SEED_FILE``. If a command is
+specified that cannot be run, no error will be reported unless
+``command_required`` is set to true.
 
 For example, to use ``pollinate`` to gather data from a
 remote entropy server and write it to ``/dev/urandom``, the following could be
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 967be168..378a6daa 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -152,6 +152,8 @@ class Distro(distros.Distro):
         elif args and isinstance(args, list):
             cmd.extend(args)
 
+        if command == "upgrade":
+            command = "-u"
         if command:
             cmd.append(command)
 
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
index 0074691b..a89e5ad2 100644
--- a/cloudinit/net/eni.py
+++ b/cloudinit/net/eni.py
@@ -387,6 +387,8 @@ class Renderer(renderer.Renderer):
                 if k == 'network':
                     if ':' in route[k]:
                         route_line += ' -A inet6'
+                    elif route.get('prefix') == 32:
+                        route_line += ' -host'
                     else:
                         route_line += ' -net'
                     if 'prefix' in route:
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 741c140a..94d9f1b9 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -16,6 +16,7 @@ from xml.dom import minidom
 
 from cloudinit import dmi
 from cloudinit import log as logging
+from cloudinit import safeyaml
 from cloudinit import sources
 from cloudinit import subp
 from cloudinit import util
@@ -47,6 +48,7 @@ LOG = logging.getLogger(__name__)
 
 CONFGROUPNAME_GUESTCUSTOMIZATION = "deployPkg"
 GUESTCUSTOMIZATION_ENABLE_CUST_SCRIPTS = "enable-custom-scripts"
+VMWARE_IMC_DIR = "/var/run/vmware-imc"
 
 
 class DataSourceOVF(sources.DataSource):
@@ -99,9 +101,7 @@ class DataSourceOVF(sources.DataSource):
             if not self.vmware_customization_supported:
                 LOG.debug("Skipping the check for "
                           "VMware Customization support")
-            elif not util.get_cfg_option_bool(
-                    self.sys_cfg, "disable_vmware_customization", True):
-
+            else:
                 search_paths = (
                     "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools",
                     "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools")
@@ -119,7 +119,9 @@ class DataSourceOVF(sources.DataSource):
                     # When the VM is powered on, the "VMware Tools" daemon
                     # copies the customization specification file to
                     # /var/run/vmware-imc directory. cloud-init code needs
-                    # to search for the file in that directory.
+                    # to search for the file in that directory which indicates
+                    # that required metadata and userdata files are now
+                    # present.
                     max_wait = get_max_wait_from_cfg(self.ds_cfg)
                     vmwareImcConfigFilePath = util.log_time(
                         logfunc=LOG.debug,
@@ -129,26 +131,83 @@ class DataSourceOVF(sources.DataSource):
                 else:
                     LOG.debug("Did not find the customization plugin.")
 
+                md_path = None
                 if vmwareImcConfigFilePath:
+                    imcdirpath = os.path.dirname(vmwareImcConfigFilePath)
+                    cf = ConfigFile(vmwareImcConfigFilePath)
+                    self._vmware_cust_conf = Config(cf)
                     LOG.debug("Found VMware Customization Config File at %s",
                               vmwareImcConfigFilePath)
-                    nicspath = wait_for_imc_cfg_file(
-                        filename="nics.txt", maxwait=10, naplen=5)
+                    try:
+                        (md_path, ud_path, nicspath) = collect_imc_file_paths(
+                            self._vmware_cust_conf)
+                    except FileNotFoundError as e:
+                        _raise_error_status(
+                            "File(s) missing in directory",
+                            e,
+                            GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED,
+                            vmwareImcConfigFilePath,
+                            self._vmware_cust_conf)
                 else:
                     LOG.debug("Did not find VMware Customization Config File")
-            else:
-                LOG.debug("Customization for VMware platform is disabled.")
 
-        if vmwareImcConfigFilePath:
+                # Honor disable_vmware_customization setting on metadata absent
+                if not md_path:
+                    if util.get_cfg_option_bool(self.sys_cfg,
+                                                "disable_vmware_customization",
+                                                True):
+                        LOG.debug(
+                            "Customization for VMware platform is disabled.")
+                        # reset vmwareImcConfigFilePath to None to avoid
+                        # customization for VMware platform
+                        vmwareImcConfigFilePath = None
+
+        use_raw_data = bool(vmwareImcConfigFilePath and md_path)
+        if use_raw_data:
+            set_gc_status(self._vmware_cust_conf, "Started")
+            LOG.debug("Start to load cloud-init meta data and user data")
+            try:
+                (md, ud, cfg, network) = load_cloudinit_data(md_path, ud_path)
+
+                if network:
+                    self._network_config = network
+                else:
+                    self._network_config = (
+                        self.distro.generate_fallback_config()
+                    )
+
+            except safeyaml.YAMLError as e:
+                _raise_error_status(
+                    "Error parsing the cloud-init meta data",
+                    e,
+                    GuestCustErrorEnum.GUESTCUST_ERROR_WRONG_META_FORMAT,
+                    vmwareImcConfigFilePath,
+                    self._vmware_cust_conf)
+            except Exception as e:
+                _raise_error_status(
+                    "Error loading cloud-init configuration",
+                    e,
+                    GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED,
+                    vmwareImcConfigFilePath,
+                    self._vmware_cust_conf)
+
+            self._vmware_cust_found = True
+            found.append('vmware-tools')
+
+            util.del_dir(imcdirpath)
+            set_customization_status(
+                GuestCustStateEnum.GUESTCUST_STATE_DONE,
+                GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS)
+            set_gc_status(self._vmware_cust_conf, "Successful")
+
+        elif vmwareImcConfigFilePath:
+            # Load configuration from vmware_imc
             self._vmware_nics_to_enable = ""
             try:
-                cf = ConfigFile(vmwareImcConfigFilePath)
-                self._vmware_cust_conf = Config(cf)
                 set_gc_status(self._vmware_cust_conf, "Started")
 
                 (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf)
                 self._vmware_nics_to_enable = get_nics_to_enable(nicspath)
-                imcdirpath = os.path.dirname(vmwareImcConfigFilePath)
                 product_marker = self._vmware_cust_conf.marker_id
                 hasmarkerfile = check_marker_exists(
                     product_marker, os.path.join(self.paths.cloud_dir, 'data'))
@@ -684,4 +743,83 @@ def _raise_error_status(prefix, error, event, config_file, conf):
     util.del_dir(os.path.dirname(config_file))
     raise error
 
+
+def load_cloudinit_data(md_path, ud_path):
+    """
+    Load the cloud-init meta data, user data, cfg and network from the
+    given files
+
+    @return: 4-tuple of configuration
+        metadata, userdata, cfg={}, network
+
+    @raises: FileNotFoundError if md_path or ud_path are absent
+    """
+    LOG.debug('load meta data from: %s: user data from: %s',
+              md_path, ud_path)
+    md = {}
+    ud = None
+    network = None
+
+    md = safeload_yaml_or_dict(util.load_file(md_path))
+
+    if 'network' in md:
+        network = md['network']
+
+    if ud_path:
+        ud = util.load_file(ud_path).replace("\r", "")
+    return md, ud, {}, network
+
+
+def safeload_yaml_or_dict(data):
+    '''
+    The meta data could be JSON or YAML. Since YAML is a strict superset of
+    JSON, we will unmarshal the data as YAML. If data is None then a new
+    dictionary is returned.
+    '''
+    if not data:
+        return {}
+    return safeyaml.load(data)
+
+
+def collect_imc_file_paths(cust_conf):
+    '''
+    collect all the other imc files.
+
+    metadata is preferred to nics.txt configuration data.
+
+    If metadata file exists because it is specified in customization
+    configuration, then metadata is required and userdata is optional.
+
+    @return a 3-tuple containing desired configuration file paths if present
+        Expected returns:
+             1. user provided metadata and userdata (md_path, ud_path, None)
+             2. user provided metadata (md_path, None, None)
+             3. user-provided network config (None, None, nics_path)
+             4. No config found (None, None, None)
+    '''
+    md_path = None
+    ud_path = None
+    nics_path = None
+    md_file = cust_conf.meta_data_name
+    if md_file:
+        md_path = os.path.join(VMWARE_IMC_DIR, md_file)
+        if not os.path.exists(md_path):
+            raise FileNotFoundError("meta data file is not found: %s"
+                                    % md_path)
+
+        ud_file = cust_conf.user_data_name
+        if ud_file:
+            ud_path = os.path.join(VMWARE_IMC_DIR, ud_file)
+            if not os.path.exists(ud_path):
+                raise FileNotFoundError("user data file is not found: %s"
+                                        % ud_path)
+    else:
+        nics_path = os.path.join(VMWARE_IMC_DIR, "nics.txt")
+        if not os.path.exists(nics_path):
+            LOG.debug('%s does not exist.', nics_path)
+            nics_path = None
+
+    return md_path, ud_path, nics_path
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py
index 7109aef3..bdfab5a0 100644
--- a/cloudinit/sources/helpers/vmware/imc/config.py
+++ b/cloudinit/sources/helpers/vmware/imc/config.py
@@ -27,6 +27,8 @@ class Config(object):
     UTC = 'DATETIME|UTC'
     POST_GC_STATUS = 'MISC|POST-GC-STATUS'
     DEFAULT_RUN_POST_SCRIPT = 'MISC|DEFAULT-RUN-POST-CUST-SCRIPT'
+    CLOUDINIT_META_DATA = 'CLOUDINIT|METADATA'
+    CLOUDINIT_USER_DATA = 'CLOUDINIT|USERDATA'
 
     def __init__(self, configFile):
         self._configFile = configFile
@@ -130,4 +132,14 @@ class Config(object):
             raise ValueError('defaultRunPostScript value should be yes/no')
         return defaultRunPostScript == 'yes'
 
+    @property
+    def meta_data_name(self):
+        """Return the name of cloud-init meta data."""
+        return self._configFile.get(Config.CLOUDINIT_META_DATA, None)
+
+    @property
+    def user_data_name(self):
+        """Return the name of cloud-init user data."""
+        return self._configFile.get(Config.CLOUDINIT_USER_DATA, None)
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
index 65ae7390..96d839b8 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
@@ -11,5 +11,6 @@ class GuestCustErrorEnum(object):
 
     GUESTCUST_ERROR_SUCCESS = 0
     GUESTCUST_ERROR_SCRIPT_DISABLED = 6
+    GUESTCUST_ERROR_WRONG_META_FORMAT = 9
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index d5113996..c08042d6 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -262,13 +262,13 @@ def extract_authorized_keys(username, sshd_cfg_file=DEF_SSHD_CFG):
 
         except (IOError, OSError):
             # Give up and use a default key filename
-            auth_key_fns.append(default_authorizedkeys_file)
+            auth_key_fns[0] = default_authorizedkeys_file
             util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in SSH "
                         "config from %r, using 'AuthorizedKeysFile' file "
                         "%r instead", DEF_SSHD_CFG, auth_key_fns[0])
 
-    # always store all the keys in the first file configured on sshd_config
-    return (auth_key_fns[0], parse_authorized_keys(auth_key_fns))
+    # always store all the keys in the user's private file
+    return (default_authorizedkeys_file, parse_authorized_keys(auth_key_fns))
 
 
 def setup_user_keys(keys, username, options=None):
diff --git a/debian/changelog b/debian/changelog
index d6e39d98..4f1d5b9e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,22 @@
+cloud-init (20.4-66-gad23441d-0ubuntu1) hirsute; urgency=medium
+
+  * New upstream snapshot.
+    - Add antonyc to .github-cla-signers (#747) [Anton Chaporgin]
+    - integration_tests: log image serial if available (#772)
+    - Revert "ssh_util: handle non-default AuthorizedKeysFile config (#586)"
+      (#775) (LP: #1911680)
+    - [VMware] Support cloudinit raw data feature (#691) [xiaofengw-vmware]
+    - net: Fix static routes to host in eni renderer (#668) [Pavel Abalikhin]
+    - .travis.yml: don't run cloud_tests in CI (#756)
+    - test_upgrade: add some missing commas (#769)
+    - cc_seed_random: update documentation and fix integration test (#771)
+      (LP: #1911227)
+    - Fix test gh-632 test to only run on NoCloud (#770) (LP: #1911230)
+    - archlinux: fix package upgrade command handling (#768) [Bao Trinh]
+    - integration_tests: add integration test for LP: #1910835 (#761)
+
+ -- Daniel Watkins <oddbloke@ubuntu.com>  Wed, 20 Jan 2021 13:15:55 -0500
+
 cloud-init (20.4-55-g4f62ae8d-0ubuntu1) hirsute; urgency=medium
 
   * New upstream snapshot.
@@ -16,7 +35,7 @@ cloud-init (20.4-55-g4f62ae8d-0ubuntu1) hirsute; urgency=medium
     - integration_tests: port ca_certs tests from cloud_tests (#732)
     - Azure: Add telemetry for poll IMDS (#741) [Johnson Shi]
     - doc: move testing section from HACKING to its own doc (#739)
-    - No longer allow integration test failures on travis (#738) [James Falcon]
+    - No longer allow integration test failures on travis (#738)
     - stale: fix error in definition (#740)
     - integration_tests: set log-cli-level to INFO by default (#737)
     - PULL_REQUEST_TEMPLATE.md: use backticks around commit message (#736)
@@ -29,44 +48,40 @@ cloud-init (20.4-55-g4f62ae8d-0ubuntu1) hirsute; urgency=medium
       [Thomas Stringer]
     - doc: fix CloudStack configuration example (#707) [Olivier Lemasle]
     - integration_tests: restrict test_lxd_bridge appropriately (#730)
-    - Add integration tests for CLI functionality (#729) [James Falcon]
-    - Integration test for gh-626 (#728) [James Falcon]
-    - Some test_upgrade fixes (#726) [James Falcon]
+    - Add integration tests for CLI functionality (#729)
+    - Integration test for gh-626 (#728)
+    - Some test_upgrade fixes (#726)
     - Ensure overriding test vars with env vars works for booleans (#727)
-      [James Falcon]
     - integration_tests: port lxd_bridge test from cloud_tests (#718)
-    - Integration test for gh-632. (#725) [James Falcon]
-    - Integration test for gh-671 (#724) [James Falcon]
+    - Integration test for gh-632. (#725)
+    - Integration test for gh-671 (#724)
     - integration-requirements.txt: bump pycloudlib commit (#723)
     - Drop unnecessary shebang from cmd/main.py (#722) [Eduardo Otubo]
-    - Integration test for LP: #1813396 and #669 (#719) [James Falcon]
+    - Integration test for LP: #1813396 and #669 (#719)
     - integration_tests: include timestamp in log output (#720)
     - integration_tests: add test for LP: #1898997 (#713)
-    - Add integration test for power_state_change module (#717) [James Falcon]
+    - Add integration test for power_state_change module (#717)
     - Update documentation for network-config-format-v2 (#701) [ggiesen]
     - sandbox CA Cert tests to not require ca-certificates (#715)
       [Eduardo Otubo]
-    - Add upgrade integration test (#693) [James Falcon]
-    - Integration test for 570 (#712) [James Falcon]
+    - Add upgrade integration test (#693)
+    - Integration test for 570 (#712)
     - Add ability to keep snapshotted images in integration tests (#711)
-      [James Falcon]
-    - Integration test for pull #586 (#706) [James Falcon]
+    - Integration test for pull #586 (#706)
     - integration_tests: introduce skipping of tests by OS (#702)
     - integration_tests: introduce IntegrationInstance.restart (#708)
     - Add lxd-vm to list of valid integration test platforms (#705)
-      [James Falcon]
     - Adding BOOTPROTO = dhcp to render sysconfig dhcp6 stateful on RHEL
       (#685) [Eduardo Otubo]
     - Delete image snapshots created for integration tests (#682)
-      [James Falcon]
     - Parametrize ssh_keys_provided integration test (#700) [lucasmoura]
     - Drop use_sudo attribute on IntegrationInstance (#694) [lucasmoura]
     - cc_apt_configure: add riscv64 as a ports arch (#687)
       [Dimitri John Ledkov]
     - cla: add xnox (#692) [Dimitri John Ledkov]
-    - Collect logs from integration test runs (#675) [James Falcon]
+    - Collect logs from integration test runs (#675)
 
- -- Daniel Watkins <oddbloke@ubuntu.com>  Mon, 11 Jan 2021 16:50:12 -0500
+ -- Daniel Watkins <oddbloke@ubuntu.com>  Mon, 11 Jan 2021 16:57:30 -0500
 
 cloud-init (20.4-0ubuntu2) hirsute; urgency=medium
 
diff --git a/tests/integration_tests/bugs/test_gh632.py b/tests/integration_tests/bugs/test_gh632.py
index cf166356..3c1f9347 100644
--- a/tests/integration_tests/bugs/test_gh632.py
+++ b/tests/integration_tests/bugs/test_gh632.py
@@ -9,6 +9,9 @@ import pytest
 from tests.integration_tests.instances import IntegrationInstance
 
 
+# With some datasource hacking, we can run this on a NoCloud instance
+@pytest.mark.lxd_container
+@pytest.mark.lxd_vm
 @pytest.mark.sru_2020_11
 def test_datasource_rbx_no_stacktrace(client: IntegrationInstance):
     client.write_to_file(
diff --git a/tests/integration_tests/bugs/test_gh668.py b/tests/integration_tests/bugs/test_gh668.py
new file mode 100644
index 00000000..a3a0c374
--- /dev/null
+++ b/tests/integration_tests/bugs/test_gh668.py
@@ -0,0 +1,37 @@
+"""Integration test for gh-668.
+
+Ensure that static route to host is working correctly.
+The original problem is specific to the ENI renderer but that test is suitable
+for all network configuration outputs.
+"""
+
+import pytest
+
+from tests.integration_tests.instances import IntegrationInstance
+
+
+DESTINATION_IP = "172.16.0.10"
+GATEWAY_IP = "10.0.0.100"
+
+NETWORK_CONFIG = """\
+version: 2
+ethernets:
+  eth0:
+    addresses: [10.0.0.10/8]
+    dhcp4: false
+    routes:
+    - to: {}/32
+      via: {}
+""".format(DESTINATION_IP, GATEWAY_IP)
+
+EXPECTED_ROUTE = "{} via {}".format(DESTINATION_IP, GATEWAY_IP)
+
+
+@pytest.mark.lxd_container
+@pytest.mark.lxd_vm
+@pytest.mark.lxd_config_dict({
+    "user.network-config": NETWORK_CONFIG,
+})
+def test_static_route_to_host(client: IntegrationInstance):
+    route = client.execute("ip route | grep {}".format(DESTINATION_IP))
+    assert route.startswith(EXPECTED_ROUTE)
diff --git a/tests/integration_tests/bugs/test_lp1910835.py b/tests/integration_tests/bugs/test_lp1910835.py
new file mode 100644
index 00000000..87f92d5e
--- /dev/null
+++ b/tests/integration_tests/bugs/test_lp1910835.py
@@ -0,0 +1,66 @@
+"""Integration test for LP: #1910835.
+
+If users do not provide an SSH key and instead ask Azure to generate a key for
+them, the key material available in the IMDS may include CRLF sequences.  Prior
+to e56b55452549cb037da0a4165154ffa494e9678a, the Azure datasource handled keys
+via a certificate, the tooling for which removed these sequences.  This test
+ensures that cloud-init does not regress support for this Azure behaviour.
+
+This test provides the SSH key configured for tests to the instance in two
+ways: firstly, with CRLFs to mimic the generated keys, via the Azure API;
+secondly, as user-data in unmodified form.  This means that even on systems
+which exhibit the bug fetching the platform's metadata, we can SSH into the SUT
+to confirm this (instead of having to assert SSH failure; there are lots of
+reasons SSH might fail).
+
+Once SSH'd in, we check that the two keys in .ssh/authorized_keys have the same
+material: if the Azure datasource has removed the CRLFs correctly, then they
+will match.
+"""
+import pytest
+
+
+USER_DATA_TMPL = """\
+#cloud-config
+ssh_authorized_keys:
+    - {}"""
+
+
+@pytest.mark.sru_2021_01
+@pytest.mark.azure
+def test_crlf_in_azure_metadata_ssh_keys(session_cloud, setup_image):
+    authorized_keys_path = "/home/{}/.ssh/authorized_keys".format(
+        session_cloud.cloud_instance.username
+    )
+    # Pass in user-data to allow us to access the instance when the normal
+    # path fails
+    key_data = session_cloud.cloud_instance.key_pair.public_key_content
+    user_data = USER_DATA_TMPL.format(key_data)
+    # Throw a CRLF into the otherwise good key data, to emulate Azure's
+    # behaviour for generated keys
+    key_data = key_data[:20] + "\r\n" + key_data[20:]
+    vm_params = {
+        "os_profile": {
+            "linux_configuration": {
+                "ssh": {
+                    "public_keys": [
+                        {"path": authorized_keys_path, "key_data": key_data}
+                    ]
+                }
+            }
+        }
+    }
+    with session_cloud.launch(
+        launch_kwargs={"vm_params": vm_params, "user_data": user_data}
+    ) as client:
+        authorized_keys = (
+            client.read_from_file(authorized_keys_path).strip().splitlines()
+        )
+        # We expect one key from the cloud, one from user-data
+        assert 2 == len(authorized_keys)
+        # And those two keys should be the same, except for a possible key
+        # comment, which Azure strips out
+        assert (
+            authorized_keys[0].rsplit(" ")[:2]
+            == authorized_keys[1].split(" ")[:2]
+        )
diff --git a/tests/integration_tests/clouds.py b/tests/integration_tests/clouds.py
index 72d77058..fc0a61d5 100644
--- a/tests/integration_tests/clouds.py
+++ b/tests/integration_tests/clouds.py
@@ -173,6 +173,9 @@ class IntegrationCloud(ABC):
                 'cloud-init version: %s',
                 instance.execute("cloud-init --version")
             )
+            serial = instance.execute("grep serial /etc/cloud/build.info")
+            if serial:
+                log.info('image serial: %s', serial.split()[1])
         return instance
 
     def get_instance(self, cloud_instance, settings=integration_settings):
diff --git a/tests/integration_tests/modules/test_seed_random_data.py b/tests/integration_tests/modules/test_seed_random_data.py
index b365fa98..f6a67c19 100644
--- a/tests/integration_tests/modules/test_seed_random_data.py
+++ b/tests/integration_tests/modules/test_seed_random_data.py
@@ -25,4 +25,4 @@ class TestSeedRandomData:
     @pytest.mark.user_data(USER_DATA)
     def test_seed_random_data(self, client):
         seed_output = client.read_from_file("/root/seed")
-        assert seed_output.strip() == "MYUb34023nD:LFDK10913jk;dfnk:Df"
+        assert seed_output.startswith("MYUb34023nD:LFDK10913jk;dfnk:Df")
diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py
index a115bea7..660d363f 100644
--- a/tests/integration_tests/test_upgrade.py
+++ b/tests/integration_tests/test_upgrade.py
@@ -26,9 +26,9 @@ def _output_to_compare(instance, file_path, netcfg_path):
         # have broken across re-constitution of a cached datasource. Some
         # platforms invalidate their datasource cache on reboot, so we run
         # it here to ensure we get a dirty run.
-        'cloud-init init'
+        'cloud-init init',
         'grep Trace /var/log/cloud-init.log',
-        'cloud-id'
+        'cloud-id',
         'cat {}'.format(netcfg_path),
         'systemd-analyze',
         'systemd-analyze blame',
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index 16773de5..dce01f5d 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -17,6 +17,7 @@ from cloudinit.helpers import Paths
 from cloudinit.sources import DataSourceOVF as dsovf
 from cloudinit.sources.helpers.vmware.imc.config_custom_script import (
     CustomScriptNotFound)
+from cloudinit.safeyaml import YAMLError
 
 MPATH = 'cloudinit.sources.DataSourceOVF.'
 
@@ -138,16 +139,29 @@ class TestDatasourceOVF(CiTestCase):
             'DEBUG: No system-product-name found', self.logs.getvalue())
 
     def test_get_data_no_vmware_customization_disabled(self):
-        """When vmware customization is disabled via sys_cfg log a message."""
+        """When cloud-init workflow for vmware is disabled via sys_cfg and
+        no meta data provided, log a message.
+        """
         paths = Paths({'cloud_dir': self.tdir})
         ds = self.datasource(
             sys_cfg={'disable_vmware_customization': True}, distro={},
             paths=paths)
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CUSTOM-SCRIPT]
+            SCRIPT-NAME = test-script
+            [MISC]
+            MARKER-ID = 12345345
+            """)
+        util.write_file(conf_file, conf_content)
         retcode = wrap_and_call(
             'cloudinit.sources.DataSourceOVF',
             {'dmi.read_dmi_data': 'vmware',
              'transport_iso9660': NOT_FOUND,
-             'transport_vmware_guestinfo': NOT_FOUND},
+             'transport_vmware_guestinfo': NOT_FOUND,
+             'util.del_dir': True,
+             'search_file': self.tdir,
+             'wait_for_imc_cfg_file': conf_file},
             ds.get_data)
         self.assertFalse(retcode, 'Expected False return from ds.get_data')
         self.assertIn(
@@ -344,6 +358,279 @@ class TestDatasourceOVF(CiTestCase):
                         'vmware (%s/seed/ovf-env.xml)' % self.tdir,
                         ds.subplatform)
 
+    def test_get_data_cloudinit_metadata_json(self):
+        """Test metadata can be loaded to cloud-init metadata and network.
+        The metadata format is json.
+        """
+        paths = Paths({'cloud_dir': self.tdir})
+        ds = self.datasource(
+            sys_cfg={'disable_vmware_customization': True}, distro={},
+            paths=paths)
+        # Prepare the conf file
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CLOUDINIT]
+            METADATA = test-meta
+            """)
+        util.write_file(conf_file, conf_content)
+        # Prepare the meta data file
+        metadata_file = self.tmp_path('test-meta', self.tdir)
+        metadata_content = dedent("""\
+            {
+              "instance-id": "cloud-vm",
+              "local-hostname": "my-host.domain.com",
+              "network": {
+                "version": 2,
+                "ethernets": {
+                  "eths": {
+                    "match": {
+                      "name": "ens*"
+                    },
+                    "dhcp4": true
+                  }
+                }
+              }
+            }
+            """)
+        util.write_file(metadata_file, metadata_content)
+
+        with mock.patch(MPATH + 'set_customization_status',
+                        return_value=('msg', b'')):
+            result = wrap_and_call(
+                'cloudinit.sources.DataSourceOVF',
+                {'dmi.read_dmi_data': 'vmware',
+                 'util.del_dir': True,
+                 'search_file': self.tdir,
+                 'wait_for_imc_cfg_file': conf_file,
+                 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''],
+                 'get_nics_to_enable': ''},
+                ds._get_data)
+
+        self.assertTrue(result)
+        self.assertEqual("cloud-vm", ds.metadata['instance-id'])
+        self.assertEqual("my-host.domain.com", ds.metadata['local-hostname'])
+        self.assertEqual(2, ds.network_config['version'])
+        self.assertTrue(ds.network_config['ethernets']['eths']['dhcp4'])
+
+    def test_get_data_cloudinit_metadata_yaml(self):
+        """Test metadata can be loaded to cloud-init metadata and network.
+        The metadata format is yaml.
+        """
+        paths = Paths({'cloud_dir': self.tdir})
+        ds = self.datasource(
+            sys_cfg={'disable_vmware_customization': True}, distro={},
+            paths=paths)
+        # Prepare the conf file
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CLOUDINIT]
+            METADATA = test-meta
+            """)
+        util.write_file(conf_file, conf_content)
+        # Prepare the meta data file
+        metadata_file = self.tmp_path('test-meta', self.tdir)
+        metadata_content = dedent("""\
+            instance-id: cloud-vm
+            local-hostname: my-host.domain.com
+            network:
+                version: 2
+                ethernets:
+                    nics:
+                        match:
+                            name: ens*
+                        dhcp4: yes
+            """)
+        util.write_file(metadata_file, metadata_content)
+
+        with mock.patch(MPATH + 'set_customization_status',
+                        return_value=('msg', b'')):
+            result = wrap_and_call(
+                'cloudinit.sources.DataSourceOVF',
+                {'dmi.read_dmi_data': 'vmware',
+                 'util.del_dir': True,
+                 'search_file': self.tdir,
+                 'wait_for_imc_cfg_file': conf_file,
+                 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''],
+                 'get_nics_to_enable': ''},
+                ds._get_data)
+
+        self.assertTrue(result)
+        self.assertEqual("cloud-vm", ds.metadata['instance-id'])
+        self.assertEqual("my-host.domain.com", ds.metadata['local-hostname'])
+        self.assertEqual(2, ds.network_config['version'])
+        self.assertTrue(ds.network_config['ethernets']['nics']['dhcp4'])
+
+    def test_get_data_cloudinit_metadata_not_valid(self):
+        """Test metadata is not JSON or YAML format.
+        """
+        paths = Paths({'cloud_dir': self.tdir})
+        ds = self.datasource(
+            sys_cfg={'disable_vmware_customization': True}, distro={},
+            paths=paths)
+
+        # Prepare the conf file
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CLOUDINIT]
+            METADATA = test-meta
+            """)
+        util.write_file(conf_file, conf_content)
+
+        # Prepare the meta data file
+        metadata_file = self.tmp_path('test-meta', self.tdir)
+        metadata_content = "[This is not json or yaml format]a=b"
+        util.write_file(metadata_file, metadata_content)
+
+        with mock.patch(MPATH + 'set_customization_status',
+                        return_value=('msg', b'')):
+            with self.assertRaises(YAMLError) as context:
+                wrap_and_call(
+                    'cloudinit.sources.DataSourceOVF',
+                    {'dmi.read_dmi_data': 'vmware',
+                     'util.del_dir': True,
+                     'search_file': self.tdir,
+                     'wait_for_imc_cfg_file': conf_file,
+                     'collect_imc_file_paths': [
+                         self.tdir + '/test-meta', '', ''
+                     ],
+                     'get_nics_to_enable': ''},
+                    ds.get_data)
+
+        self.assertIn("expected '<document start>', but found '<scalar>'",
+                      str(context.exception))
+
+    def test_get_data_cloudinit_metadata_not_found(self):
+        """Test metadata file can't be found.
+        """
+        paths = Paths({'cloud_dir': self.tdir})
+        ds = self.datasource(
+            sys_cfg={'disable_vmware_customization': True}, distro={},
+            paths=paths)
+        # Prepare the conf file
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CLOUDINIT]
+            METADATA = test-meta
+            """)
+        util.write_file(conf_file, conf_content)
+        # Don't prepare the meta data file
+
+        with mock.patch(MPATH + 'set_customization_status',
+                        return_value=('msg', b'')):
+            with self.assertRaises(FileNotFoundError) as context:
+                wrap_and_call(
+                    'cloudinit.sources.DataSourceOVF',
+                    {'dmi.read_dmi_data': 'vmware',
+                     'util.del_dir': True,
+                     'search_file': self.tdir,
+                     'wait_for_imc_cfg_file': conf_file,
+                     'get_nics_to_enable': ''},
+                    ds.get_data)
+
+        self.assertIn('is not found', str(context.exception))
+
+    def test_get_data_cloudinit_userdata(self):
+        """Test user data can be loaded to cloud-init user data.
+        """
+        paths = Paths({'cloud_dir': self.tdir})
+        ds = self.datasource(
+            sys_cfg={'disable_vmware_customization': False}, distro={},
+            paths=paths)
+
+        # Prepare the conf file
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CLOUDINIT]
+            METADATA = test-meta
+            USERDATA = test-user
+            """)
+        util.write_file(conf_file, conf_content)
+
+        # Prepare the meta data file
+        metadata_file = self.tmp_path('test-meta', self.tdir)
+        metadata_content = dedent("""\
+            instance-id: cloud-vm
+            local-hostname: my-host.domain.com
+            network:
+                version: 2
+                ethernets:
+                    nics:
+                        match:
+                            name: ens*
+                        dhcp4: yes
+            """)
+        util.write_file(metadata_file, metadata_content)
+
+        # Prepare the user data file
+        userdata_file = self.tmp_path('test-user', self.tdir)
+        userdata_content = "This is the user data"
+        util.write_file(userdata_file, userdata_content)
+
+        with mock.patch(MPATH + 'set_customization_status',
+                        return_value=('msg', b'')):
+            result = wrap_and_call(
+                'cloudinit.sources.DataSourceOVF',
+                {'dmi.read_dmi_data': 'vmware',
+                 'util.del_dir': True,
+                 'search_file': self.tdir,
+                 'wait_for_imc_cfg_file': conf_file,
+                 'collect_imc_file_paths': [self.tdir + '/test-meta',
+                                            self.tdir + '/test-user', ''],
+                 'get_nics_to_enable': ''},
+                ds._get_data)
+
+        self.assertTrue(result)
+        self.assertEqual("cloud-vm", ds.metadata['instance-id'])
+        self.assertEqual(userdata_content, ds.userdata_raw)
+
+    def test_get_data_cloudinit_userdata_not_found(self):
+        """Test userdata file can't be found.
+        """
+        paths = Paths({'cloud_dir': self.tdir})
+        ds = self.datasource(
+            sys_cfg={'disable_vmware_customization': True}, distro={},
+            paths=paths)
+
+        # Prepare the conf file
+        conf_file = self.tmp_path('test-cust', self.tdir)
+        conf_content = dedent("""\
+            [CLOUDINIT]
+            METADATA = test-meta
+            USERDATA = test-user
+            """)
+        util.write_file(conf_file, conf_content)
+
+        # Prepare the meta data file
+        metadata_file = self.tmp_path('test-meta', self.tdir)
+        metadata_content = dedent("""\
+            instance-id: cloud-vm
+            local-hostname: my-host.domain.com
+            network:
+                version: 2
+                ethernets:
+                    nics:
+                        match:
+                            name: ens*
+                        dhcp4: yes
+            """)
+        util.write_file(metadata_file, metadata_content)
+
+        # Don't prepare the user data file
+
+        with mock.patch(MPATH + 'set_customization_status',
+                        return_value=('msg', b'')):
+            with self.assertRaises(FileNotFoundError) as context:
+                wrap_and_call(
+                    'cloudinit.sources.DataSourceOVF',
+                    {'dmi.read_dmi_data': 'vmware',
+                     'util.del_dir': True,
+                     'search_file': self.tdir,
+                     'wait_for_imc_cfg_file': conf_file,
+                     'get_nics_to_enable': ''},
+                    ds.get_data)
+
+        self.assertIn('is not found', str(context.exception))
+
 
 class TestTransportIso9660(CiTestCase):
 
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index bf0cdabb..38d934d4 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -4820,6 +4820,9 @@ class TestEniRoundTrip(CiTestCase):
             {'type': 'route', 'id': 6,
              'metric': 1, 'destination': '10.0.200.0/16',
              'gateway': '172.23.31.1'},
+            {'type': 'route', 'id': 7,
+             'metric': 1, 'destination': '10.0.0.100/32',
+             'gateway': '172.23.31.1'},
         ]
 
         files = self._render_and_read(
@@ -4843,6 +4846,10 @@ class TestEniRoundTrip(CiTestCase):
              '172.23.31.1 metric 1 || true'),
             ('pre-down route del -net 10.0.200.0/16 gw '
              '172.23.31.1 metric 1 || true'),
+            ('post-up route add -host 10.0.0.100/32 gw '
+             '172.23.31.1 metric 1 || true'),
+            ('pre-down route del -host 10.0.0.100/32 gw '
+             '172.23.31.1 metric 1 || true'),
         ]
         found = files['/etc/network/interfaces'].splitlines()
 
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index 88a111e3..fd1d1bac 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -593,7 +593,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase):
             fpw.pw_name, sshd_config)
         content = ssh_util.update_authorized_keys(auth_key_entries, [])
 
-        self.assertEqual(authorized_keys, auth_key_fn)
+        self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn)
         self.assertTrue(VALID_CONTENT['rsa'] in content)
         self.assertTrue(VALID_CONTENT['dsa'] in content)
 
@@ -610,7 +610,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase):
         sshd_config = self.tmp_path('sshd_config')
         util.write_file(
             sshd_config,
-            "AuthorizedKeysFile %s %s" % (user_keys, authorized_keys)
+            "AuthorizedKeysFile %s %s" % (authorized_keys, user_keys)
         )
 
         (auth_key_fn, auth_key_entries) = ssh_util.extract_authorized_keys(
@@ -618,7 +618,7 @@ class TestMultipleSshAuthorizedKeysFile(test_helpers.CiTestCase):
         )
         content = ssh_util.update_authorized_keys(auth_key_entries, [])
 
-        self.assertEqual(user_keys, auth_key_fn)
+        self.assertEqual("%s/.ssh/authorized_keys" % fpw.pw_dir, auth_key_fn)
         self.assertTrue(VALID_CONTENT['rsa'] in content)
         self.assertTrue(VALID_CONTENT['dsa'] in content)
 
diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py
index 9c7d25fa..430cc69f 100644
--- a/tests/unittests/test_vmware_config_file.py
+++ b/tests/unittests/test_vmware_config_file.py
@@ -525,5 +525,21 @@ class TestVmwareNetConfig(CiTestCase):
                    'gateway': '10.20.87.253'}]}],
             nc.generate())
 
+    def test_meta_data(self):
+        cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg")
+        conf = Config(cf)
+        self.assertIsNone(conf.meta_data_name)
+        cf._insertKey("CLOUDINIT|METADATA", "test-metadata")
+        conf = Config(cf)
+        self.assertEqual("test-metadata", conf.meta_data_name)
+
+    def test_user_data(self):
+        cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg")
+        conf = Config(cf)
+        self.assertIsNone(conf.user_data_name)
+        cf._insertKey("CLOUDINIT|USERDATA", "test-userdata")
+        conf = Config(cf)
+        self.assertEqual("test-userdata", conf.user_data_name)
+
 
 # vi: ts=4 expandtab
diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
index d6ca6d1b..4cee82ac 100644
--- a/tools/.github-cla-signers
+++ b/tools/.github-cla-signers
@@ -2,6 +2,7 @@ ader1990
 ajmyyra
 AlexBaranowski
 Aman306
+antonyc
 aswinrajamannar
 beezly
 bipinbachhao
@@ -26,11 +27,13 @@ nishigori
 olivierlemasle
 omBratteng
 onitake
+qubidt
 riedel
 slyon
 smoser
 sshedi
 TheRealFalcon
+tnt-dev
 tomponline
 tsanghan
 WebSpider
diff --git a/tox.ini b/tox.ini
index 7c71f0f6..9444e66c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -184,5 +184,6 @@ markers =
     user_data: the user data to be passed to the test instance
     instance_name: the name to be used for the test instance
     sru_2020_11: test is part of the 2020/11 SRU verification
+    sru_2021_01: test is part of the 2021/01 SRU verification
     ubuntu: this test should run on Ubuntu
     unstable: skip this test because it is flakey