merge from 5f7825e at 20.2-45-g5f7825e2

28 files changed, 335 insertions, 78 deletions

diff --git a/.travis.yml b/.travis.yml
index 3de1066b..84f38533 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -79,8 +79,6 @@ matrix:
               PYTEST_ADDOPTS=-v  # List all tests run by pytest
           dist: xenial
         - python: 3.6
-          env: TOXENV=pycodestyle
-        - python: 3.6
-          env: TOXENV=pyflakes
+          env: TOXENV=flake8
         - python: 3.6
           env: TOXENV=pylint
diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py
index 6d36a4ea..cb15b2d2 100644
--- a/cloudinit/cmd/tests/test_query.py
+++ b/cloudinit/cmd/tests/test_query.py
@@ -261,7 +261,7 @@ class TestQuery(CiTestCase):
         args = self.args(
             debug=False, dump_all=False, format=None,
             instance_data=self.instance_data, list_keys=True, user_data='ud',
-            vendor_data='vd',  varname='top')
+            vendor_data='vd', varname='top')
         with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 with mock.patch('os.getuid') as m_getuid:
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 9a33451d..b1c7b471 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -297,7 +297,7 @@ schema = {
                 },
                 'conf': {
                     'type': 'string',
-                    'description':  dedent("""\
+                    'description': dedent("""\
                         Specify configuration for apt, such as proxy
                         configuration. This configuration is specified as a
                         string. For multiline apt configuration, make sure
diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py
index 01d61fa1..03285ef0 100644
--- a/cloudinit/config/cc_chef.py
+++ b/cloudinit/config/cc_chef.py
@@ -50,6 +50,7 @@ file).
                            written to /etc/chef/client.rb)
 
     chef:
+      chef_license:
       client_key:
       encrypted_data_bag_secret:
       environment:
@@ -125,6 +126,7 @@ CHEF_RB_TPL_PATH_KEYS = frozenset([
     'file_cache_path',
     'pid_file',
     'encrypted_data_bag_secret',
+    'chef_license',
 ])
 CHEF_RB_TPL_KEYS = list(CHEF_RB_TPL_DEFAULTS.keys())
 CHEF_RB_TPL_KEYS.extend(CHEF_RB_TPL_BOOL_KEYS)
diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py
index a323edfa..7888464e 100644
--- a/cloudinit/config/cc_grub_dpkg.py
+++ b/cloudinit/config/cc_grub_dpkg.py
@@ -1,8 +1,9 @@
-# Copyright (C) 2009-2010 Canonical Ltd.
+# Copyright (C) 2009-2010, 2020 Canonical Ltd.
 # Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
 #
 # Author: Scott Moser <scott.moser@canonical.com>
 # Author: Juerg Haefliger <juerg.haefliger@hp.com>
+# Author: Matthew Ruffell <matthew.ruffell@canonical.com>
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
@@ -15,15 +16,15 @@ Configure which device is used as the target for grub installation. This module
 should work correctly by default without any user configuration. It can be
 enabled/disabled using the ``enabled`` config key in the ``grub_dpkg`` config
 dict. The global config key ``grub-dpkg`` is an alias for ``grub_dpkg``. If no
-installation device is specified this module will look for the first existing
-device in:
+installation device is specified this module will execute grub-probe to
+determine which disk the /boot directory is associated with.
 
-    - ``/dev/sda``
-    - ``/dev/vda``
-    - ``/dev/xvda``
-    - ``/dev/sda1``
-    - ``/dev/vda1``
-    - ``/dev/xvda1``
+The value which is placed into the debconf database is in the format which the
+grub postinstall script expects. Normally, this is a /dev/disk/by-id/ value,
+but we do fallback to the plain disk name if a by-id name is not present.
+
+If this module is executed inside a container, then the debconf database is
+seeded with empty values, and install_devices_empty is set to true.
 
 **Internal name:** ``cc_grub_dpkg``
 
@@ -43,10 +44,66 @@ device in:
 import os
 
 from cloudinit import util
+from cloudinit.util import ProcessExecutionError
 
 distros = ['ubuntu', 'debian']
 
 
+def fetch_idevs(log):
+    """
+    Fetches the /dev/disk/by-id device grub is installed to.
+    Falls back to plain disk name if no by-id entry is present.
+    """
+    disk = ""
+    devices = []
+
+    try:
+        # get the root disk where the /boot directory resides.
+        disk = util.subp(['grub-probe', '-t', 'disk', '/boot'],
+                         capture=True)[0].strip()
+    except ProcessExecutionError as e:
+        # grub-common may not be installed, especially on containers
+        # FileNotFoundError is a nested exception of ProcessExecutionError
+        if isinstance(e.reason, FileNotFoundError):
+            log.debug("'grub-probe' not found in $PATH")
+        # disks from the container host are present in /proc and /sys
+        # which is where grub-probe determines where /boot is.
+        # it then checks for existence in /dev, which fails as host disks
+        # are not exposed to the container.
+        elif "failed to get canonical path" in e.stderr:
+            log.debug("grub-probe 'failed to get canonical path'")
+        else:
+            # something bad has happened, continue to log the error
+            raise
+    except Exception:
+        util.logexc(log, "grub-probe failed to execute for grub-dpkg")
+
+    if not disk or not os.path.exists(disk):
+        # If we failed to detect a disk, we can return early
+        return ''
+
+    try:
+        # check if disk exists and use udevadm to fetch symlinks
+        devices = util.subp(
+            ['udevadm', 'info', '--root', '--query=symlink', disk],
+            capture=True
+        )[0].strip().split()
+    except Exception:
+        util.logexc(
+            log, "udevadm DEVLINKS symlink query failed for disk='%s'", disk
+        )
+
+    log.debug('considering these device symlinks: %s', ','.join(devices))
+    # filter symlinks for /dev/disk/by-id entries
+    devices = [dev for dev in devices if 'disk/by-id' in dev]
+    log.debug('filtered to these disk/by-id symlinks: %s', ','.join(devices))
+    # select first device if there is one, else fall back to plain name
+    idevs = sorted(devices)[0] if devices else disk
+    log.debug('selected %s', idevs)
+
+    return idevs
+
+
 def handle(name, cfg, _cloud, log, _args):
 
     mycfg = cfg.get("grub_dpkg", cfg.get("grub-dpkg", {}))
@@ -62,22 +119,10 @@ def handle(name, cfg, _cloud, log, _args):
     idevs_empty = util.get_cfg_option_str(
         mycfg, "grub-pc/install_devices_empty", None)
 
-    if ((os.path.exists("/dev/sda1") and not os.path.exists("/dev/sda")) or
-       (os.path.exists("/dev/xvda1") and not os.path.exists("/dev/xvda"))):
-        if idevs is None:
-            idevs = ""
-        if idevs_empty is None:
-            idevs_empty = "true"
-    else:
-        if idevs_empty is None:
-            idevs_empty = "false"
-        if idevs is None:
-            idevs = "/dev/sda"
-            for dev in ("/dev/sda", "/dev/vda", "/dev/xvda",
-                        "/dev/sda1", "/dev/vda1", "/dev/xvda1"):
-                if os.path.exists(dev):
-                    idevs = dev
-                    break
+    if idevs is None:
+        idevs = fetch_idevs(log)
+    if idevs_empty is None:
+        idevs_empty = "false" if idevs else "true"
 
     # now idevs and idevs_empty are set to determined values
     # or, those set by user
diff --git a/cloudinit/config/tests/test_grub_dpkg.py b/cloudinit/config/tests/test_grub_dpkg.py
new file mode 100644
index 00000000..01efa330
--- /dev/null
+++ b/cloudinit/config/tests/test_grub_dpkg.py
@@ -0,0 +1,176 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+
+import pytest
+
+from unittest import mock
+from logging import Logger
+from cloudinit.util import ProcessExecutionError
+from cloudinit.config.cc_grub_dpkg import fetch_idevs, handle
+
+
+class TestFetchIdevs:
+    """Tests cc_grub_dpkg.fetch_idevs()"""
+
+    # Note: udevadm info returns devices in a large single line string
+    @pytest.mark.parametrize(
+        "grub_output,path_exists,expected_log_call,udevadm_output"
+        ",expected_idevs",
+        [
+            # Inside a container, grub not installed
+            (
+                ProcessExecutionError(reason=FileNotFoundError()),
+                False,
+                mock.call("'grub-probe' not found in $PATH"),
+                '',
+                '',
+            ),
+            # Inside a container, grub installed
+            (
+                ProcessExecutionError(stderr="failed to get canonical path"),
+                False,
+                mock.call("grub-probe 'failed to get canonical path'"),
+                '',
+                '',
+            ),
+            # KVM Instance
+            (
+                ['/dev/vda'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-path/pci-0000:00:00.0 ',
+                    '/dev/disk/by-path/virtio-pci-0000:00:00.0 '
+                ),
+                '/dev/vda',
+            ),
+            # Xen Instance
+            (
+                ['/dev/xvda'],
+                True,
+                None,
+                '',
+                '/dev/xvda',
+            ),
+            # NVMe Hardware Instance
+            (
+                ['/dev/nvme1n1'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-id/nvme-Company_hash000 ',
+                    '/dev/disk/by-id/nvme-nvme.000-000-000-000-000 ',
+                    '/dev/disk/by-path/pci-0000:00:00.0-nvme-0 '
+                ),
+                '/dev/disk/by-id/nvme-Company_hash000',
+            ),
+            # SCSI Hardware Instance
+            (
+                ['/dev/sda'],
+                True,
+                None,
+                (
+                    '/dev/disk/by-id/company-user-1 ',
+                    '/dev/disk/by-id/scsi-0Company_user-1 ',
+                    '/dev/disk/by-path/pci-0000:00:00.0-scsi-0:0:0:0 '
+                ),
+                '/dev/disk/by-id/company-user-1',
+            ),
+        ],
+    )
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.os.path.exists")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    def test_fetch_idevs(self, m_subp, m_exists, m_logexc, grub_output,
+                         path_exists, expected_log_call, udevadm_output,
+                         expected_idevs):
+        """Tests outputs from grub-probe and udevadm info against grub-dpkg"""
+        m_subp.side_effect = [
+            grub_output,
+            ["".join(udevadm_output)]
+        ]
+        m_exists.return_value = path_exists
+        log = mock.Mock(spec=Logger)
+        idevs = fetch_idevs(log)
+        assert expected_idevs == idevs
+        if expected_log_call is not None:
+            assert expected_log_call in log.debug.call_args_list
+
+
+class TestHandle:
+    """Tests cc_grub_dpkg.handle()"""
+
+    @pytest.mark.parametrize(
+        "cfg_idevs,cfg_idevs_empty,fetch_idevs_output,expected_log_output",
+        [
+            (
+                # No configuration
+                None,
+                None,
+                '/dev/disk/by-id/nvme-Company_hash000',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/disk/by-id/nvme-Company_hash000','false'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty unset
+                '/dev/sda',
+                None,
+                '/dev/sda',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/sda','false'"
+                ),
+            ),
+            (
+                # idevs unset, idevs_empty set
+                None,
+                'true',
+                '/dev/xvda',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/xvda','true'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty set
+                '/dev/vda',
+                'false',
+                '/dev/disk/by-id/company-user-1',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/vda','false'"
+                ),
+            ),
+            (
+                # idevs set, idevs_empty set
+                # Respect what the user defines, even if its logically wrong
+                '/dev/nvme0n1',
+                'true',
+                '',
+                (
+                    "Setting grub debconf-set-selections with ",
+                    "'/dev/nvme0n1','true'"
+                ),
+            )
+        ],
+    )
+    @mock.patch("cloudinit.config.cc_grub_dpkg.fetch_idevs")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.get_cfg_option_str")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.logexc")
+    @mock.patch("cloudinit.config.cc_grub_dpkg.util.subp")
+    def test_handle(self, m_subp, m_logexc, m_get_cfg_str, m_fetch_idevs,
+                    cfg_idevs, cfg_idevs_empty, fetch_idevs_output,
+                    expected_log_output):
+        """Test setting of correct debconf database entries"""
+        m_get_cfg_str.side_effect = [
+            cfg_idevs,
+            cfg_idevs_empty
+        ]
+        m_fetch_idevs.return_value = fetch_idevs_output
+        log = mock.Mock(spec=Logger)
+        handle(mock.Mock(), mock.Mock(), mock.Mock(), log, mock.Mock())
+        log.debug.assert_called_with("".join(expected_log_output))
+
+
+# vi: ts=4 expandtab
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index e99529df..35a10590 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -582,7 +582,7 @@ class Distro(metaclass=abc.ABCMeta):
         # passwd must use short '-l' due to SLES11 lacking long form '--lock'
         lock_tools = (['passwd', '-l', name], ['usermod', '--lock', name])
         try:
-            cmd = next(l for l in lock_tools if util.which(l[0]))
+            cmd = next(tool for tool in lock_tools if util.which(tool[0]))
         except StopIteration:
             raise RuntimeError((
                 "Unable to lock user account '%s'. No tools available. "
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index cb8c1601..8af24fa9 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -824,13 +824,13 @@ def get_interfaces_by_mac_on_freebsd():
     # flatten each interface block in a single line
     def flatten(out):
         curr_block = ''
-        for l in out.split('\n'):
-            if l.startswith('\t'):
-                curr_block += l
+        for line in out.split('\n'):
+            if line.startswith('\t'):
+                curr_block += line
             else:
                 if curr_block:
                     yield curr_block
-                curr_block = l
+                curr_block = line
         yield curr_block
 
     # looks for interface and mac in a list of flatten block
diff --git a/cloudinit/net/tests/test_dhcp.py b/cloudinit/net/tests/test_dhcp.py
index bc7bef45..7768da7c 100644
--- a/cloudinit/net/tests/test_dhcp.py
+++ b/cloudinit/net/tests/test_dhcp.py
@@ -211,7 +211,7 @@ class TestDHCPParseStaticRoutes(CiTestCase):
             "class_b": "16,172,16,10",
             "class_a": "8,10,10",
             "gateway": "0,0",
-            "netlen":  "33,0",
+            "netlen": "33,0",
         }
         for rfc3442 in bad_rfc3442.values():
             self.assertEqual([], parse_static_routes(rfc3442))
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 6cbfbbac..0ec5f6ec 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -116,7 +116,7 @@ def _write_host_key_to_guest_attributes(key_type, key_value):
     resp = url_helper.readurl(url=url, data=key_value, headers=HEADERS,
                               request_method='PUT', check_status=False)
     if resp.ok():
-        LOG.debug('Wrote %s host key to guest attributes.',  key_type)
+        LOG.debug('Wrote %s host key to guest attributes.', key_type)
     else:
         LOG.debug('Unable to write %s host key to guest attributes.', key_type)
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 02c9a7b8..a08ab404 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -417,9 +417,9 @@ def read_context_disk_dir(source_dir, asuser=None):
 
     if ssh_key_var:
         lines = context.get(ssh_key_var).splitlines()
-        results['metadata']['public-keys'] = [l for l in lines
-                                              if len(l) and not
-                                              l.startswith("#")]
+        results['metadata']['public-keys'] = [
+            line for line in lines if len(line) and not line.startswith("#")
+        ]
 
     # custom hostname -- try hostname or leave cloud-init
     # itself create hostname from IP address later
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index e91398ea..a4373f24 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -411,8 +411,11 @@ class ConfigDriveReader(BaseReader):
         keydata = meta_js.get('public-keys', keydata)
         if keydata:
             lines = keydata.splitlines()
-            md['public-keys'] = [l for l in lines
-                                 if len(l) and not l.startswith("#")]
+            md['public-keys'] = [
+                line
+                for line in lines
+                if len(line) and not line.startswith("#")
+            ]
 
         # config-drive-v1 has no way for openstack to provide the instance-id
         # so we copy that into metadata from the user input
@@ -674,11 +677,13 @@ def convert_net_json(network_json=None, known_macs=None):
                 raise ValueError("Unable to find a system nic for %s" % d)
             d['name'] = known_macs[mac]
 
-        for cfg, key, fmt, target in link_updates:
-            if isinstance(target, (list, tuple)):
-                cfg[key] = [fmt % link_id_info[l]['name'] for l in target]
+        for cfg, key, fmt, targets in link_updates:
+            if isinstance(targets, (list, tuple)):
+                cfg[key] = [
+                    fmt % link_id_info[target]['name'] for target in targets
+                ]
             else:
-                cfg[key] = fmt % link_id_info[target]['name']
+                cfg[key] = fmt % link_id_info[targets]['name']
 
     # Infiniband interfaces may be referenced in network_data.json by a 6 byte
     # Ethernet MAC-style address, and we use that address to look up the
diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py
index c3a9b5b7..918c4aec 100644
--- a/cloudinit/ssh_util.py
+++ b/cloudinit/ssh_util.py
@@ -344,7 +344,9 @@ def update_ssh_config(updates, fname=DEF_SSHD_CFG):
     changed = update_ssh_config_lines(lines=lines, updates=updates)
     if changed:
         util.write_file(
-            fname, "\n".join([str(l) for l in lines]) + "\n", copy_mode=True)
+            fname, "\n".join(
+                [str(line) for line in lines]
+            ) + "\n", copy_mode=True)
     return len(changed) != 0
 
 
diff --git a/cloudinit/tests/test_url_helper.py b/cloudinit/tests/test_url_helper.py
index 29b39374..364ec822 100644
--- a/cloudinit/tests/test_url_helper.py
+++ b/cloudinit/tests/test_url_helper.py
@@ -85,7 +85,7 @@ class TestReadFileOrUrl(CiTestCase):
         read_file_or_url(url, headers=headers, headers_redact=['sensitive'])
         logs = self.logs.getvalue()
         for k in headers.keys():
-            self.assertEqual(headers[k],  httpretty.last_request().headers[k])
+            self.assertEqual(headers[k], httpretty.last_request().headers[k])
         self.assertIn(REDACTED, logs)
         self.assertNotIn('sekret', logs)
 
diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt
index 39957e58..bb4b058c 100644
--- a/doc/examples/cloud-config-chef.txt
+++ b/doc/examples/cloud-config-chef.txt
@@ -52,6 +52,9 @@ apt:
 
 chef:
 
+  # Valid values are 'accept' and 'accept-no-persist'
+  chef_license: "accept"
+  
   # Valid values are 'gems' and 'packages' and 'omnibus'
   install_type: "packages"
 
diff --git a/templates/chef_client.rb.tmpl b/templates/chef_client.rb.tmpl
index 99978d3b..0a759b04 100644
--- a/templates/chef_client.rb.tmpl
+++ b/templates/chef_client.rb.tmpl
@@ -14,6 +14,9 @@ you need to add the following to config:
 The reason these are not in quotes is because they are ruby
 symbols that will be placed inside here, and not actual strings...
 #}
+{% if chef_license %}
+chef_license          "{{chef_license}}"
+{% endif%}
 {% if log_level %}
 log_level              {{log_level}}
 {% endif %}
diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py
index 529e79cd..efc35c7f 100644
--- a/tests/cloud_tests/platforms/instances.py
+++ b/tests/cloud_tests/platforms/instances.py
@@ -132,8 +132,8 @@ class Instance(TargetBase):
         """
         def clean_test(test):
             """Clean formatting for system ready test testcase."""
-            return ' '.join(l for l in test.strip().splitlines()
-                            if not l.lstrip().startswith('#'))
+            return ' '.join(line for line in test.strip().splitlines()
+                            if not line.lstrip().startswith('#'))
 
         boot_timeout = self.config['boot_timeout']
         tests = [self.config['system_ready_script']]
diff --git a/tests/cloud_tests/releases.yaml b/tests/cloud_tests/releases.yaml
index 187f3ac3..e76a3d35 100644
--- a/tests/cloud_tests/releases.yaml
+++ b/tests/cloud_tests/releases.yaml
@@ -133,6 +133,22 @@ features:
 
 releases:
     # UBUNTU =================================================================
+    groovy:
+        # EOL: Jul 2021
+        default:
+            enabled: true
+            release: groovy
+            version: "20.10"
+            os: ubuntu
+            feature_groups:
+                - base
+                - debian_base
+                - ubuntu_specific
+        lxd:
+            sstreams_server: https://cloud-images.ubuntu.com/daily
+            alias: groovy
+            setup_overrides: null
+            override_templates: false
     focal:
         # EOL: Apr 2025
         default:
diff --git a/tests/cloud_tests/testcases/base.py b/tests/cloud_tests/testcases/base.py
index 7b67f54e..68d59111 100644
--- a/tests/cloud_tests/testcases/base.py
+++ b/tests/cloud_tests/testcases/base.py
@@ -141,8 +141,8 @@ class CloudTestCase(unittest.TestCase):
     def test_no_warnings_in_log(self):
         """Unexpected warnings should not be found in the log."""
         warnings = [
-            l for l in self.get_data_file('cloud-init.log').splitlines()
-            if 'WARN' in l]
+            line for line in self.get_data_file('cloud-init.log').splitlines()
+            if 'WARN' in line]
         joined_warnings = '\n'.join(warnings)
         for expected_warning in self.expected_warnings:
             self.assertIn(
diff --git a/tests/unittests/test_datasource/test_ec2.py b/tests/unittests/test_datasource/test_ec2.py
index d413d1cd..ad1ea595 100644
--- a/tests/unittests/test_datasource/test_ec2.py
+++ b/tests/unittests/test_datasource/test_ec2.py
@@ -612,7 +612,9 @@ class TestEc2(test_helpers.HttprettyTestCase):
         for log in expected_logs:
             self.assertIn(log, logs)
         self.assertEqual(
-            1, len([l for l in logs.splitlines() if failed_put_log in l]))
+            1,
+            len([line for line in logs.splitlines() if failed_put_log in line])
+        )
 
     def test_aws_token_redacted(self):
         """Verify that aws tokens are redacted when logged."""
diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py
index a19c35c8..486a2345 100644
--- a/tests/unittests/test_datasource/test_ovf.py
+++ b/tests/unittests/test_datasource/test_ovf.py
@@ -48,7 +48,7 @@ def fill_properties(props, template=OVF_ENV_CONTENT):
     for key, val in props.items():
         lines.append(prop_tmpl.format(key=key, val=val))
     indent = "        "
-    properties = ''.join([indent + l + "\n" for l in lines])
+    properties = ''.join([indent + line + "\n" for line in lines])
     return template.format(properties=properties)
 
 
diff --git a/tests/unittests/test_ds_identify.py b/tests/unittests/test_ds_identify.py
index f0e96b44..c2318570 100644
--- a/tests/unittests/test_ds_identify.py
+++ b/tests/unittests/test_ds_identify.py
@@ -611,8 +611,10 @@ class TestDsIdentify(DsIdentifyBase):
         ret = self._check_via_dict(
             cust, RC_FOUND,
             func=".", args=[os.path.join(rootd, mpp)], rootd=rootd)
-        line = [l for l in ret.stdout.splitlines() if l.startswith(pre)][0]
-        toks = line.replace(pre, "").split(":")
+        match = [
+            line for line in ret.stdout.splitlines() if line.startswith(pre)
+        ][0]
+        toks = match.replace(pre, "").split(":")
         expected = ["/sbin", "/bin", "/usr/sbin", "/usr/bin", "/mycust/path"]
         self.assertEqual(expected, [p for p in expected if p in toks],
                          "path did not have expected tokens")
diff --git a/tests/unittests/test_handler/test_handler_apt_source_v3.py b/tests/unittests/test_handler/test_handler_apt_source_v3.py
index 4762dbef..aefe26c4 100644
--- a/tests/unittests/test_handler/test_handler_apt_source_v3.py
+++ b/tests/unittests/test_handler/test_handler_apt_source_v3.py
@@ -1033,7 +1033,9 @@ class TestDebconfSelections(TestCase):
         # assumes called with *args value.
         selections = m_set_sel.call_args_list[0][0][0].decode()
 
-        missing = [l for l in lines if l not in selections.splitlines()]
+        missing = [
+            line for line in lines if line not in selections.splitlines()
+        ]
         self.assertEqual([], missing)
 
     @mock.patch("cloudinit.config.cc_apt_configure.dpkg_reconfigure")
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index 513c18b5..8c476418 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -130,6 +130,7 @@ class TestChef(FilesystemMockingTestCase):
 
         # This should create a file of the format...
         # Created by cloud-init v. 0.7.6 on Sat, 11 Oct 2014 23:57:21 +0000
+        chef_license           "accept"
         log_level              :info
         ssl_verify_mode        :verify_none
         log_location           "/var/log/chef/client.log"
@@ -153,6 +154,7 @@ class TestChef(FilesystemMockingTestCase):
         util.write_file('/etc/cloud/templates/chef_client.rb.tmpl', tpl_file)
         cfg = {
             'chef': {
+                'chef_license': "accept",
                 'server_url': 'localhost',
                 'validation_name': 'bob',
                 'validation_key': "/etc/chef/vkey.pem",
diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py
index e075a64c..84d3a5f0 100644
--- a/tests/unittests/test_net.py
+++ b/tests/unittests/test_net.py
@@ -944,7 +944,7 @@ NETWORK_CONFIGS = {
                         dhcp6: true
         """).rstrip(' '),
         'expected_sysconfig_opensuse': {
-            'ifcfg-iface0':  textwrap.dedent("""\
+            'ifcfg-iface0': textwrap.dedent("""\
                 BOOTPROTO=dhcp
                 DHCLIENT6_MODE=managed
                 STARTMODE=auto""")
@@ -1028,7 +1028,7 @@ NETWORK_CONFIGS = {
     },
     'v6_and_v4': {
         'expected_sysconfig_opensuse': {
-            'ifcfg-iface0':  textwrap.dedent("""\
+            'ifcfg-iface0': textwrap.dedent("""\
                 BOOTPROTO=dhcp
                 DHCLIENT6_MODE=managed
                 STARTMODE=auto""")
diff --git a/tests/unittests/test_sshutil.py b/tests/unittests/test_sshutil.py
index 0be41924..b4767f0c 100644
--- a/tests/unittests/test_sshutil.py
+++ b/tests/unittests/test_sshutil.py
@@ -299,7 +299,7 @@ class TestUpdateSshConfigLines(test_helpers.CiTestCase):
         lines = ssh_util.parse_ssh_config_lines(list(self.exlines))
         result = ssh_util.update_ssh_config_lines(lines, updates)
         self.assertEqual([], result)
-        self.assertEqual(self.exlines, [str(l) for l in lines])
+        self.assertEqual(self.exlines, [str(line) for line in lines])
 
     def test_keycase_not_modified(self):
         """Original case of key should not be changed on update.
diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers
index 2c924f85..c3113705 100644
--- a/tools/.github-cla-signers
+++ b/tools/.github-cla-signers
@@ -2,6 +2,7 @@ beezly
 bipinbachhao
 dhensby
 lucasmoura
+matthewruffell
 nishigori
 tomponline
 TheRealFalcon
diff --git a/tox.ini b/tox.ini
index 95a8511f..5ee67368 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py3, xenial-dev, pycodestyle, pyflakes, pylint
+envlist = py3, xenial-dev, flake8, pylint
 recreate = True
 
 [testenv]
@@ -9,11 +9,11 @@ setenv =
 passenv=
     PYTEST_ADDOPTS
 
-[testenv:pycodestyle]
+[testenv:flake8]
 basepython = python3
 deps =
-    pycodestyle==2.4.0
-commands = {envpython} -m pycodestyle {posargs:cloudinit/ tests/ tools/}
+    flake8==3.8.2
+commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/}
 
 # https://github.com/gabrielfalcao/HTTPretty/issues/223
 setenv =
@@ -43,9 +43,16 @@ basepython = python2.7
 deps = -r{toxinidir}/test-requirements.txt
 
 [flake8]
-#H102  Apache 2.0 license header not found
-ignore=H404,H405,H105,H301,H104,H403,H101,H102,H106,H304
+# E121: continuation line under-indented for hanging indent
+# E123: closing bracket does not match indentation of opening bracket’s line
+# E126: continuation line over-indented for hanging indent
+# E226: missing whitespace around arithmetic operator
+# W503: line break before binary operator
+# W504: line break after binary operator
+ignore=E121,E123,E126,E226,W503,W504
 exclude = .venv,.tox,dist,doc,*egg,.git,build,tools
+per-file-ignores =
+    cloudinit/cmd/main.py:E402
 
 [testenv:doc]
 basepython = python3
@@ -109,18 +116,9 @@ deps =
     jsonpatch==1.16
     pytest==3.0.7
 
-[testenv:tip-pycodestyle]
-commands = {envpython} -m pycodestyle {posargs:cloudinit/ tests/ tools/}
-deps = pycodestyle
-
-[testenv:pyflakes]
-commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/}
-deps =
-    pyflakes==2.1.1
-
-[testenv:tip-pyflakes]
-commands = {envpython} -m pyflakes {posargs:cloudinit/ tests/ tools/}
-deps = pyflakes
+[testenv:tip-flake8]
+commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/}
+deps = flake8
 
 [testenv:tip-pylint]
 commands = {envpython} -m pylint {posargs:cloudinit tests tools}