diff options
27 files changed, 400 insertions, 343 deletions
diff --git a/cloudinit/analyze/show.py b/cloudinit/analyze/show.py index abfa0913..04621f12 100644 --- a/cloudinit/analyze/show.py +++ b/cloudinit/analyze/show.py @@ -8,7 +8,6 @@ import base64 import datetime import json import os -import sys import time from cloudinit import subp, util @@ -257,25 +256,21 @@ def gather_timestamps_using_systemd(): status = SUCCESS_CODE # lxc based containers do not set their monotonic zero point to be when # the container starts, instead keep using host boot as zero point - # time.CLOCK_MONOTONIC_RAW is only available in python 3.3 if util.is_container(): # clock.monotonic also uses host boot as zero point - if sys.version_info >= (3, 3): - base_time = float(time.time()) - float(time.monotonic()) - # TODO: lxcfs automatically truncates /proc/uptime to seconds - # in containers when https://github.com/lxc/lxcfs/issues/292 - # is fixed, util.uptime() should be used instead of stat on - try: - file_stat = os.stat("/proc/1/cmdline") - kernel_start = file_stat.st_atime - except OSError as err: - raise RuntimeError( - "Could not determine container boot " - "time from /proc/1/cmdline. ({})".format(err) - ) from err - status = CONTAINER_CODE - else: - status = FAIL_CODE + base_time = float(time.time()) - float(time.monotonic()) + # TODO: lxcfs automatically truncates /proc/uptime to seconds + # in containers when https://github.com/lxc/lxcfs/issues/292 + # is fixed, util.uptime() should be used instead of stat on + try: + file_stat = os.stat("/proc/1/cmdline") + kernel_start = file_stat.st_atime + except OSError as err: + raise RuntimeError( + "Could not determine container boot " + "time from /proc/1/cmdline. ({})".format(err) + ) from err + status = CONTAINER_CODE kernel_end = base_time + delta_k_end cloudinit_sysd = base_time + delta_ci_s diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index c0b073b5..2e964dcf 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -257,7 +257,6 @@ def handle(name, cfg, cloud, log, _args): # (TODO(harlowja) is this really needed??) cleaned_lines = [i.lstrip() for i in contents.splitlines()] cleaned_contents = "\n".join(cleaned_lines) - # Move to puppet_config.read_file when dropping py2.7 puppet_config.read_file( StringIO(cleaned_contents), source=p_constants.conf_path ) diff --git a/cloudinit/config/cc_ubuntu_drivers.py b/cloudinit/config/cc_ubuntu_drivers.py index 15f621a7..a962bce3 100644 --- a/cloudinit/config/cc_ubuntu_drivers.py +++ b/cloudinit/config/cc_ubuntu_drivers.py @@ -5,6 +5,14 @@ import os from textwrap import dedent +try: + import debconf + + HAS_DEBCONF = True +except ImportError: + debconf = None + HAS_DEBCONF = False + from cloudinit import log as logging from cloudinit import subp, temp_utils, type_utils, util from cloudinit.config.schema import MetaSchema, get_meta_doc @@ -48,10 +56,6 @@ OLD_UBUNTU_DRIVERS_STDERR_NEEDLE = ( # 'linux-restricted-modules' deb to accept the NVIDIA EULA and the package # will automatically link the drivers to the running kernel. -# EOL_XENIAL: can then drop this script and use python3-debconf which is only -# available in Bionic and later. Can't use python3-debconf currently as it -# isn't in Xenial and doesn't yet support X_LOADTEMPLATEFILE debconf command. - NVIDIA_DEBCONF_CONTENT = """\ Template: linux/nvidia/latelink Type: boolean @@ -61,13 +65,8 @@ Description: Late-link NVIDIA kernel modules? make them available for use. """ -NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT = """\ -#!/bin/sh -# Allow cloud-init to trigger EULA acceptance via registering a debconf -# template to set linux/nvidia/latelink true -. /usr/share/debconf/confmodule -db_x_loadtemplatefile "$1" cloud-init -""" + +X_LOADTEMPLATEFILE = "X_LOADTEMPLATEFILE" def install_drivers(cfg, pkg_install_func): @@ -108,15 +107,10 @@ def install_drivers(cfg, pkg_install_func): # Register and set debconf selection linux/nvidia/latelink = true tdir = temp_utils.mkdtemp(needs_exe=True) debconf_file = os.path.join(tdir, "nvidia.template") - debconf_script = os.path.join(tdir, "nvidia-debconf.sh") try: util.write_file(debconf_file, NVIDIA_DEBCONF_CONTENT) - util.write_file( - debconf_script, - util.encode_text(NVIDIA_DRIVER_LATELINK_DEBCONF_SCRIPT), - mode=0o755, - ) - subp.subp([debconf_script, debconf_file]) + with debconf.DebconfCommunicator("cloud-init") as dc: + dc.command(X_LOADTEMPLATEFILE, debconf_file) except Exception as e: util.logexc( LOG, "Failed to register NVIDIA debconf template: %s", str(e) @@ -143,5 +137,11 @@ def handle(name, cfg, cloud, log, _args): if "drivers" not in cfg: log.debug("Skipping module named %s, no 'drivers' key in config", name) return + if not HAS_DEBCONF: + log.warning( + "Skipping module named %s, 'python3-debconf' is not installed", + name, + ) + return install_drivers(cfg["drivers"], cloud.distro.install_packages) diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 8fd82427..b6e2c549 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -7,6 +7,7 @@ import logging import os import re import sys +import textwrap import typing from collections import defaultdict from copy import deepcopy @@ -563,9 +564,7 @@ def _get_examples(meta: MetaSchema) -> str: return "" rst_content = SCHEMA_EXAMPLES_HEADER for count, example in enumerate(examples): - # Python2.6 is missing textwrapper.indent - lines = example.split("\n") - indented_lines = [" {0}".format(line) for line in lines] + indented_lines = textwrap.indent(example, " ").split("\n") if rst_content != SCHEMA_EXAMPLES_HEADER: indented_lines.insert( 0, SCHEMA_EXAMPLES_SPACER_TEMPLATE.format(count + 1) diff --git a/cloudinit/distros/netbsd.py b/cloudinit/distros/netbsd.py index c0d6390f..b3232feb 100644 --- a/cloudinit/distros/netbsd.py +++ b/cloudinit/distros/netbsd.py @@ -89,15 +89,6 @@ class NetBSD(cloudinit.distros.bsd.BSD): def set_passwd(self, user, passwd, hashed=False): if hashed: hashed_pw = passwd - elif not hasattr(crypt, "METHOD_BLOWFISH"): - # crypt.METHOD_BLOWFISH comes with Python 3.7 which is available - # on NetBSD 7 and 8. - LOG.error( - "Cannot set non-encrypted password for user %s. " - "Python >= 3.7 is required.", - user, - ) - return else: method = crypt.METHOD_BLOWFISH # pylint: disable=E1101 hashed_pw = crypt.crypt(passwd, crypt.mksalt(method)) diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index ec6470a9..4e75b6ec 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -11,7 +11,6 @@ import copy -from cloudinit import util from cloudinit.distros import PREFERRED_NTP_CLIENTS, debian @@ -39,14 +38,7 @@ class Distro(debian.Distro): def preferred_ntp_clients(self): """The preferred ntp client is dependent on the version.""" if not self._preferred_ntp_clients: - (_name, _version, codename) = util.system_info()["dist"] - # Xenial cloud-init only installed ntp, UbuntuCore has timesyncd. - if codename == "xenial" and not util.system_is_snappy(): - self._preferred_ntp_clients = ["ntp"] - else: - self._preferred_ntp_clients = copy.deepcopy( - PREFERRED_NTP_CLIENTS - ) + self._preferred_ntp_clients = copy.deepcopy(PREFERRED_NTP_CLIENTS) return self._preferred_ntp_clients diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py index 9438bbca..66ad598f 100644 --- a/cloudinit/net/netplan.py +++ b/cloudinit/net/netplan.py @@ -2,6 +2,7 @@ import copy import os +import textwrap from typing import cast from cloudinit import log as logging @@ -430,7 +431,7 @@ class Renderer(renderer.Renderer): explicit_end=False, noalias=True, ) - txt = util.indent(dump, " " * 4) + txt = textwrap.indent(dump, " " * 4) return [txt] return [] diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index 7417a26e..2c64e492 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -57,7 +57,7 @@ NET_CONFIG_TO_V2: Dict[str, Dict[str, Any]] = { "bond-miimon": "mii-monitor-interval", "bond-min-links": "min-links", "bond-mode": "mode", - "bond-num-grat-arp": "gratuitious-arp", + "bond-num-grat-arp": "gratuitous-arp", "bond-primary": "primary", "bond-primary-reselect": "primary-reselect-policy", "bond-updelay": "up-delay", @@ -796,13 +796,12 @@ class NetworkStateInterpreter(metaclass=CommandHandlerMeta): for (key, value) in item_cfg.items() if key not in NETWORK_V2_KEY_FILTER ) - # we accept the fixed spelling, but write the old for compatibility - # Xenial does not have an updated netplan which supports the - # correct spelling. LP: #1756701 + # We accept both spellings (as netplan does). LP: #1756701 + # Normalize internally to the new spelling: params = item_params.get("parameters", {}) - grat_value = params.pop("gratuitous-arp", None) + grat_value = params.pop("gratuitious-arp", None) if grat_value: - params["gratuitious-arp"] = grat_value + params["gratuitous-arp"] = grat_value v1_cmd = { "type": cmd_type, diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index e63e223d..b18d8d3f 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -285,7 +285,6 @@ BUILTIN_DS_CONFIG = { "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, "apply_network_config": True, # Use IMDS published network configuration } -# RELEASE_BLOCKER: Xenial and earlier apply_network_config default is False BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG = { "disk_setup": { @@ -1742,8 +1741,7 @@ def address_ephemeral_resize( try: os.unlink(sempath) LOG.debug("%s removed.", bmsg) - except Exception as e: - # python3 throws FileNotFoundError, python2 throws OSError + except FileNotFoundError as e: LOG.warning("%s: remove failed! (%s)", bmsg, e) else: LOG.debug("%s did not exist.", bmsg) @@ -2087,9 +2085,8 @@ def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): seed = util.load_file(source, quiet=True, decode=False) # The seed generally contains non-Unicode characters. load_file puts - # them into a str (in python 2) or bytes (in python 3). In python 2, - # bad octets in a str cause util.json_dumps() to throw an exception. In - # python 3, bytes is a non-serializable type, and the handler load_file + # them into bytes (in python 3). + # bytes is a non-serializable type, and the handler load_file # uses applies b64 encoding *again* to handle it. The simplest solution # is to just b64encode the data and then decode it to a serializable # string. Same number of bits of entropy, just with 25% more zeroes. diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 9b15190b..68ac1ba7 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -477,12 +477,6 @@ class DataSourceEc2(sources.DataSource): ), ) - # RELEASE_BLOCKER: xenial should drop the below if statement, - # because the issue being addressed doesn't exist pre-netplan. - # (This datasource doesn't implement check_instance_id() so the - # datasource object is recreated every boot; this means we don't - # need to modify update_events on cloud-init upgrade.) - # Non-VPC (aka Classic) Ec2 instances need to rewrite the # network config file every boot due to MAC address change. if self.is_classic_instance(): diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index d2603900..11168f6a 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -713,8 +713,7 @@ class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): if self.is_b64_encoded(key): try: val = base64.b64decode(val.encode()).decode() - # Bogus input produces different errors in Python 2 and 3 - except (TypeError, binascii.Error): + except binascii.Error: LOG.warning("Failed base64 decoding key '%s': %s", key, val) if strip: diff --git a/cloudinit/util.py b/cloudinit/util.py index 0e215218..aad8607d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1741,37 +1741,15 @@ def json_serialize_default(_obj): return "Warning: redacted unserializable type {0}".format(type(_obj)) -def json_preserialize_binary(data): - """Preserialize any discovered binary values to avoid json.dumps issues. - - Used only on python 2.7 where default type handling is not honored for - failure to encode binary data. LP: #1801364. - TODO(Drop this function when py2.7 support is dropped from cloud-init) - """ - data = obj_copy.deepcopy(data) - for key, value in data.items(): - if isinstance(value, (dict)): - data[key] = json_preserialize_binary(value) - if isinstance(value, bytes): - data[key] = "ci-b64:{0}".format(b64e(value)) - return data - - def json_dumps(data): """Return data in nicely formatted json.""" - try: - return json.dumps( - data, - indent=1, - sort_keys=True, - separators=(",", ": "), - default=json_serialize_default, - ) - except UnicodeDecodeError: - if sys.version_info[:2] == (2, 7): - data = json_preserialize_binary(data) - return json.dumps(data) - raise + return json.dumps( + data, + indent=1, + sort_keys=True, + separators=(",", ": "), + default=json_serialize_default, + ) def ensure_dir(path, mode=None): @@ -2828,14 +2806,6 @@ def system_is_snappy(): return False -def indent(text, prefix): - """replacement for indent from textwrap that is not available in 2.7.""" - lines = [] - for line in text.splitlines(True): - lines.append(prefix + line) - return "".join(lines) - - def rootdev_from_cmdline(cmdline): found = None for tok in cmdline.split(): diff --git a/doc/rtd/topics/network-config-format-v2.rst b/doc/rtd/topics/network-config-format-v2.rst index c1bf05d1..3080c6d4 100644 --- a/doc/rtd/topics/network-config-format-v2.rst +++ b/doc/rtd/topics/network-config-format-v2.rst @@ -338,7 +338,7 @@ Set whether to set all slaves to the same MAC address when adding them to the bond, or how else the system should handle MAC addresses. The possible values are ``none``, ``active``, and ``follow``. -**gratuitious-arp**: <*(scalar)>* +**gratuitous-arp**: <*(scalar)>* Specify how many ARP packets to send after failover. Once a link is up on a new slave, a notification is sent and possibly repeated if diff --git a/integration-requirements.txt b/integration-requirements.txt index 7b64554d..cd10c540 100644 --- a/integration-requirements.txt +++ b/integration-requirements.txt @@ -1,5 +1,5 @@ # PyPI requirements for cloud-init integration testing # https://cloudinit.readthedocs.io/en/latest/topics/integration_tests.html # -pycloudlib @ git+https://github.com/canonical/pycloudlib.git@c42341990cb35460946ee04e2623d0f9fffe2b3c +pycloudlib @ git+https://github.com/canonical/pycloudlib.git@6eee33c9c4f630bc9c13b6e48f9ab36e7fb79ca6 pytest diff --git a/packages/bddeb b/packages/bddeb index b009021a..fdb541d4 100755 --- a/packages/bddeb +++ b/packages/bddeb @@ -34,7 +34,13 @@ DEBUILD_ARGS = ["-S", "-d"] def get_release_suffix(release): - """Given ubuntu release (xenial), return a suffix for package (~16.04.1)""" + """Given ubuntu release, return a suffix for package + + Examples: + --------- + >>> get_release_suffix("jammy") + '~22.04.1' + """ csv_path = "/usr/share/distro-info/ubuntu.csv" rels = {} # fields are version, codename, series, created, release, eol, eol-server @@ -150,10 +156,6 @@ def get_parser(): default=False, action='store_true') - parser.add_argument("--python2", dest="python2", - help=("build debs for python2 rather than python3"), - default=False, action='store_true') - parser.add_argument("--init-system", dest="init_system", help=("build deb with INIT_SYSTEM=xxx" " (default: %(default)s"), diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in index 1491822b..917e9516 100644 --- a/packages/redhat/cloud-init.spec.in +++ b/packages/redhat/cloud-init.spec.in @@ -48,11 +48,6 @@ BuildRequires: {{r}} Requires: dmidecode %endif -# python2.6 needs argparse -%if "%{?el6}" == "1" -Requires: python-argparse -%endif - # Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json. # Install them as BuildRequires too as they're used for testing. diff --git a/pyproject.toml b/pyproject.toml index acec6cc8..2ee26121 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ module = [ "BaseHTTPServer", "configobj", "cloudinit.feature_overrides", + "debconf", "httpretty", "httplib", "jsonpatch", @@ -91,9 +91,8 @@ def render_tmpl(template, mode=None): in that file if user had something there. b.) debuild will complain that files are different outside of the debian directory.""" - # older versions of tox use bdist (xenial), and then install from there. # newer versions just use install. - if not (sys.argv[1] == "install" or sys.argv[1].startswith("bdist*")): + if not (sys.argv[1] == "install"): return template tmpl_ext = ".tmpl" diff --git a/tests/integration_tests/bugs/test_lp1835584.py b/tests/integration_tests/bugs/test_lp1835584.py index 765d73ef..8ecb1246 100644 --- a/tests/integration_tests/bugs/test_lp1835584.py +++ b/tests/integration_tests/bugs/test_lp1835584.py @@ -12,8 +12,6 @@ In cases where product_uuid changes case, ensure cloud-init doesn't recreate ssh hostkeys across reboot (due to detecting an instance_id change). This currently only affects linux-azure-fips -> linux-azure on Bionic. -This test won't run on Xenial because both linux-azure-fips and linux-azure -report uppercase product_uuids. The test will launch a specific Bionic Ubuntu PRO FIPS image which has a linux-azure-fips kernel known to report product_uuid as uppercase. Then upgrade diff --git a/tests/integration_tests/modules/test_ubuntu_drivers.py b/tests/integration_tests/modules/test_ubuntu_drivers.py new file mode 100644 index 00000000..4fbfba3c --- /dev/null +++ b/tests/integration_tests/modules/test_ubuntu_drivers.py @@ -0,0 +1,37 @@ +import re + +import pytest + +from tests.integration_tests.clouds import IntegrationCloud +from tests.integration_tests.util import verify_clean_log + +USER_DATA = """\ +#cloud-config +drivers: + nvidia: + license-accepted: true +""" + +# NOTE(VM.GPU2.1 is not in all availability_domains: use qIZq:US-ASHBURN-AD-1) + + +@pytest.mark.adhoc # Expensive instance type +@pytest.mark.oci +def test_ubuntu_drivers_installed(session_cloud: IntegrationCloud): + with session_cloud.launch( + launch_kwargs={"instance_type": "VM.GPU2.1"}, user_data=USER_DATA + ) as client: + log = client.read_from_file("/var/log/cloud-init.log") + verify_clean_log(log) + assert 1 == log.count( + "Installing and activating NVIDIA drivers " + "(nvidia/license-accepted=True, version=latest)" + ) + result = client.execute("dpkg -l | grep nvidia") + assert result.ok, "No nvidia packages found" + assert re.search( + r"ii\s+linux-modules-nvidia-\d+-server", result.stdout + ), ( + f"Did not find specific nvidia drivers packages in:" + f" {result.stdout}" + ) diff --git a/tests/unittests/config/test_cc_ntp.py b/tests/unittests/config/test_cc_ntp.py index c2bce2a3..41b5fb9b 100644 --- a/tests/unittests/config/test_cc_ntp.py +++ b/tests/unittests/config/test_cc_ntp.py @@ -499,15 +499,6 @@ class TestNtp(FilesystemMockingTestCase): expected_client = mycloud.distro.preferred_ntp_clients[0] self.assertEqual("chrony", expected_client) - @mock.patch("cloudinit.util.system_info") - def test_ubuntu_xenial_picks_ntp(self, m_sysinfo): - """Test Ubuntu picks ntp on xenial release""" - - m_sysinfo.return_value = {"dist": ("Ubuntu", "16.04", "xenial")} - mycloud = self._get_cloud("ubuntu") - expected_client = mycloud.distro.preferred_ntp_clients[0] - self.assertEqual("ntp", expected_client) - @mock.patch("cloudinit.config.cc_ntp.subp.which") def test_snappy_system_picks_timesyncd(self, m_which): """Test snappy systems prefer installed clients""" diff --git a/tests/unittests/config/test_cc_ubuntu_drivers.py b/tests/unittests/config/test_cc_ubuntu_drivers.py index b814f21a..9d54467e 100644 --- a/tests/unittests/config/test_cc_ubuntu_drivers.py +++ b/tests/unittests/config/test_cc_ubuntu_drivers.py @@ -6,6 +6,7 @@ import re import pytest +from cloudinit import log from cloudinit.config import cc_ubuntu_drivers as drivers from cloudinit.config.schema import ( SchemaValidationError, @@ -13,7 +14,7 @@ from cloudinit.config.schema import ( validate_cloudconfig_schema, ) from cloudinit.subp import ProcessExecutionError -from tests.unittests.helpers import CiTestCase, mock, skipUnlessJsonSchema +from tests.unittests.helpers import mock, skipUnlessJsonSchema MPATH = "cloudinit.config.cc_ubuntu_drivers." M_TMP_PATH = MPATH + "temp_utils.mkdtemp" @@ -31,223 +32,286 @@ OLD_UBUNTU_DRIVERS_ERROR_STDERR = ( # pylint: disable=no-value-for-parameter -class AnyTempScriptAndDebconfFile(object): - def __init__(self, tmp_dir, debconf_file): - self.tmp_dir = tmp_dir - self.debconf_file = debconf_file - - def __eq__(self, cmd): - if not len(cmd) == 2: - return False - script, debconf_file = cmd - if bool(script.startswith(self.tmp_dir) and script.endswith(".sh")): - return debconf_file == self.debconf_file - return False - - -class TestUbuntuDrivers(CiTestCase): - cfg_accepted: dict = {"drivers": {"nvidia": {"license-accepted": True}}} +@pytest.mark.parametrize( + "cfg_accepted,install_gpgpu", + [ + pytest.param( + {"drivers": {"nvidia": {"license-accepted": True}}}, + ["ubuntu-drivers", "install", "--gpgpu", "nvidia"], + id="without_version", + ), + pytest.param( + { + "drivers": { + "nvidia": {"license-accepted": True, "version": "123"} + } + }, + ["ubuntu-drivers", "install", "--gpgpu", "nvidia:123"], + id="with_version", + ), + ], +) +@mock.patch(MPATH + "debconf") +@mock.patch(MPATH + "HAS_DEBCONF", True) +class TestUbuntuDrivers: install_gpgpu = ["ubuntu-drivers", "install", "--gpgpu", "nvidia"] - with_logs = True - + @pytest.mark.parametrize( + "true_value", + [ + True, + "yes", + "true", + "on", + "1", + ], + ) @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=False) - def _assert_happy_path_taken(self, config, m_which, m_subp, m_tmp): + def test_happy_path_taken( + self, + m_which, + m_subp, + m_tmp, + m_debconf, + tmpdir, + cfg_accepted, + install_gpgpu, + true_value, + ): """Positive path test through handle. Package should be installed.""" - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") + new_config: dict = copy.deepcopy(cfg_accepted) + new_config["drivers"]["nvidia"]["license-accepted"] = true_value + + tdir = tmpdir + debconf_file = tdir.join("nvidia.template") m_tmp.return_value = tdir myCloud = mock.MagicMock() - drivers.handle("ubuntu_drivers", config, myCloud, None, None) - self.assertEqual( - [mock.call(["ubuntu-drivers-common"])], - myCloud.distro.install_packages.call_args_list, - ) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, - ) - - def test_handle_does_package_install(self): - self._assert_happy_path_taken(self.cfg_accepted) - - def test_trueish_strings_are_considered_approval(self): - for true_value in ["yes", "true", "on", "1"]: - new_config = copy.deepcopy(self.cfg_accepted) - new_config["drivers"]["nvidia"]["license-accepted"] = true_value - self._assert_happy_path_taken(new_config) + drivers.handle("ubuntu_drivers", new_config, myCloud, None, None) + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers-common"]) + ] == myCloud.distro.install_packages.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp") @mock.patch(MPATH + "subp.which", return_value=False) def test_handle_raises_error_if_no_drivers_found( - self, m_which, m_subp, m_tmp + self, + m_which, + m_subp, + m_tmp, + m_debconf, + caplog, + tmpdir, + cfg_accepted, + install_gpgpu, ): """If ubuntu-drivers doesn't install any drivers, raise an error.""" - tdir = self.tmp_dir() + tdir = tmpdir debconf_file = os.path.join(tdir, "nvidia.template") m_tmp.return_value = tdir myCloud = mock.MagicMock() - def fake_subp(cmd): - if cmd[0].startswith(tdir): - return - raise ProcessExecutionError( - stdout="No drivers found for installation.\n", exit_code=1 - ) - - m_subp.side_effect = fake_subp - - with self.assertRaises(Exception): - drivers.handle( - "ubuntu_drivers", self.cfg_accepted, myCloud, None, None - ) - self.assertEqual( - [mock.call(["ubuntu-drivers-common"])], - myCloud.distro.install_packages.call_args_list, - ) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, + m_subp.side_effect = ProcessExecutionError( + stdout="No drivers found for installation.\n", exit_code=1 ) - self.assertIn( - "ubuntu-drivers found no drivers for installation", - self.logs.getvalue(), + + with pytest.raises(Exception): + drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, None, None) + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers-common"]) + ] == myCloud.distro.install_packages.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list + assert ( + "ubuntu-drivers found no drivers for installation" in caplog.text ) + @pytest.mark.parametrize( + "config", + [ + pytest.param( + {"drivers": {"nvidia": {"license-accepted": False}}}, + id="license_not_accepted", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "garbage"}}}, + id="garbage_in_license_field", + ), + pytest.param({"drivers": {"nvidia": {}}}, id="no_license_key"), + pytest.param( + {"drivers": {"acme": {"license-accepted": True}}}, + id="no_nvidia_key", + ), + # ensure we don't do anything if string refusal given + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "no"}}}, + id="string_given_no", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "false"}}}, + id="string_given_false", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "off"}}}, + id="string_given_off", + ), + pytest.param( + {"drivers": {"nvidia": {"license-accepted": "0"}}}, + id="string_given_0", + ), + # specifying_a_version_doesnt_override_license_acceptance + pytest.param( + { + "drivers": { + "nvidia": {"license-accepted": False, "version": "123"} + } + }, + id="with_version", + ), + ], + ) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=False) - def _assert_inert_with_config(self, config, m_which, m_subp): + def test_handle_inert( + self, m_which, m_subp, m_debconf, cfg_accepted, install_gpgpu, config + ): """Helper to reduce repetition when testing negative cases""" myCloud = mock.MagicMock() drivers.handle("ubuntu_drivers", config, myCloud, None, None) - self.assertEqual(0, myCloud.distro.install_packages.call_count) - self.assertEqual(0, m_subp.call_count) - - def test_handle_inert_if_license_not_accepted(self): - """Ensure we don't do anything if the license is rejected.""" - self._assert_inert_with_config( - {"drivers": {"nvidia": {"license-accepted": False}}} - ) - - def test_handle_inert_if_garbage_in_license_field(self): - """Ensure we don't do anything if unknown text is in license field.""" - self._assert_inert_with_config( - {"drivers": {"nvidia": {"license-accepted": "garbage"}}} - ) - - def test_handle_inert_if_no_license_key(self): - """Ensure we don't do anything if no license key.""" - self._assert_inert_with_config({"drivers": {"nvidia": {}}}) - - def test_handle_inert_if_no_nvidia_key(self): - """Ensure we don't do anything if other license accepted.""" - self._assert_inert_with_config( - {"drivers": {"acme": {"license-accepted": True}}} - ) - - def test_handle_inert_if_string_given(self): - """Ensure we don't do anything if string refusal given.""" - for false_value in ["no", "false", "off", "0"]: - self._assert_inert_with_config( - {"drivers": {"nvidia": {"license-accepted": false_value}}} - ) + assert 0 == myCloud.distro.install_packages.call_count + assert 0 == m_subp.call_count @mock.patch(MPATH + "install_drivers") - def test_handle_no_drivers_does_nothing(self, m_install_drivers): + def test_handle_no_drivers_does_nothing( + self, m_install_drivers, m_debconf, cfg_accepted, install_gpgpu + ): """If no 'drivers' key in the config, nothing should be done.""" myCloud = mock.MagicMock() myLog = mock.MagicMock() drivers.handle("ubuntu_drivers", {"foo": "bzr"}, myCloud, myLog, None) - self.assertIn( - "Skipping module named", myLog.debug.call_args_list[0][0][0] - ) - self.assertEqual(0, m_install_drivers.call_count) + assert "Skipping module named" in myLog.debug.call_args_list[0][0][0] + assert 0 == m_install_drivers.call_count @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=True) def test_install_drivers_no_install_if_present( - self, m_which, m_subp, m_tmp + self, + m_which, + m_subp, + m_tmp, + m_debconf, + tmpdir, + cfg_accepted, + install_gpgpu, ): """If 'ubuntu-drivers' is present, no package install should occur.""" - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") + tdir = tmpdir + debconf_file = tmpdir.join("nvidia.template") m_tmp.return_value = tdir pkg_install = mock.MagicMock() drivers.install_drivers( - self.cfg_accepted["drivers"], pkg_install_func=pkg_install + cfg_accepted["drivers"], pkg_install_func=pkg_install ) - self.assertEqual(0, pkg_install.call_count) - self.assertEqual([mock.call("ubuntu-drivers")], m_which.call_args_list) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, - ) - - def test_install_drivers_rejects_invalid_config(self): + assert 0 == pkg_install.call_count + assert [mock.call("ubuntu-drivers")] == m_which.call_args_list + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list + + def test_install_drivers_rejects_invalid_config( + self, m_debconf, cfg_accepted, install_gpgpu + ): """install_drivers should raise TypeError if not given a config dict""" pkg_install = mock.MagicMock() - with self.assertRaisesRegex(TypeError, ".*expected dict.*"): + with pytest.raises(TypeError, match=".*expected dict.*"): drivers.install_drivers("mystring", pkg_install_func=pkg_install) - self.assertEqual(0, pkg_install.call_count) + assert 0 == pkg_install.call_count @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp") @mock.patch(MPATH + "subp.which", return_value=False) def test_install_drivers_handles_old_ubuntu_drivers_gracefully( - self, m_which, m_subp, m_tmp + self, + m_which, + m_subp, + m_tmp, + m_debconf, + caplog, + tmpdir, + cfg_accepted, + install_gpgpu, ): """Older ubuntu-drivers versions should emit message and raise error""" - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") - m_tmp.return_value = tdir + debconf_file = tmpdir.join("nvidia.template") + m_tmp.return_value = tmpdir myCloud = mock.MagicMock() - def fake_subp(cmd): - if cmd[0].startswith(tdir): - return - raise ProcessExecutionError( - stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2 - ) + m_subp.side_effect = ProcessExecutionError( + stderr=OLD_UBUNTU_DRIVERS_ERROR_STDERR, exit_code=2 + ) - m_subp.side_effect = fake_subp + with pytest.raises(Exception): + drivers.handle("ubuntu_drivers", cfg_accepted, myCloud, None, None) + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers-common"]) + ] == myCloud.distro.install_packages.call_args_list + assert [mock.call(install_gpgpu)] == m_subp.call_args_list + assert ( + MPATH[:-1], + log.WARNING, + ( + "the available version of ubuntu-drivers is" + " too old to perform requested driver installation" + ), + ) == caplog.record_tuples[-1] - with self.assertRaises(Exception): + @mock.patch(M_TMP_PATH) + @mock.patch(MPATH + "subp.subp", return_value=("", "")) + @mock.patch(MPATH + "subp.which", return_value=False) + def test_debconf_not_installed_does_nothing( + self, + m_which, + m_subp, + m_tmp, + m_debconf, + tmpdir, + cfg_accepted, + install_gpgpu, + ): + m_debconf.DebconfCommunicator.side_effect = AttributeError + m_tmp.return_value = tmpdir + myCloud = mock.MagicMock() + version_none_cfg = { + "drivers": {"nvidia": {"license-accepted": True, "version": None}} + } + with pytest.raises(AttributeError): drivers.handle( - "ubuntu_drivers", self.cfg_accepted, myCloud, None, None + "ubuntu_drivers", version_none_cfg, myCloud, None, None ) - self.assertEqual( - [mock.call(["ubuntu-drivers-common"])], - myCloud.distro.install_packages.call_args_list, - ) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(self.install_gpgpu), - ], - m_subp.call_args_list, - ) - self.assertIn( - "WARNING: the available version of ubuntu-drivers is" - " too old to perform requested driver installation", - self.logs.getvalue(), + assert ( + 0 == m_debconf.DebconfCommunicator.__enter__().command.call_count ) + assert 0 == m_subp.call_count + +@mock.patch(MPATH + "debconf") +@mock.patch(MPATH + "HAS_DEBCONF", True) +class TestUbuntuDriversWithVersion: + """With-version specific tests""" -# Sub-class TestUbuntuDrivers to run the same test cases, but with a version -class TestUbuntuDriversWithVersion(TestUbuntuDrivers): cfg_accepted = { "drivers": {"nvidia": {"license-accepted": True, "version": "123"}} } @@ -256,30 +320,76 @@ class TestUbuntuDriversWithVersion(TestUbuntuDrivers): @mock.patch(M_TMP_PATH) @mock.patch(MPATH + "subp.subp", return_value=("", "")) @mock.patch(MPATH + "subp.which", return_value=False) - def test_version_none_uses_latest(self, m_which, m_subp, m_tmp): - tdir = self.tmp_dir() - debconf_file = os.path.join(tdir, "nvidia.template") - m_tmp.return_value = tdir + def test_version_none_uses_latest( + self, m_which, m_subp, m_tmp, m_debconf, tmpdir + ): + debconf_file = tmpdir.join("nvidia.template") + m_tmp.return_value = tmpdir myCloud = mock.MagicMock() version_none_cfg = { "drivers": {"nvidia": {"license-accepted": True, "version": None}} } drivers.handle("ubuntu_drivers", version_none_cfg, myCloud, None, None) - self.assertEqual( - [ - mock.call(AnyTempScriptAndDebconfFile(tdir, debconf_file)), - mock.call(["ubuntu-drivers", "install", "--gpgpu", "nvidia"]), - ], - m_subp.call_args_list, + assert [ + mock.call(drivers.X_LOADTEMPLATEFILE, debconf_file) + ] == m_debconf.DebconfCommunicator().__enter__().command.call_args_list + assert [ + mock.call(["ubuntu-drivers", "install", "--gpgpu", "nvidia"]), + ] == m_subp.call_args_list + + +@mock.patch(MPATH + "debconf") +class TestUbuntuDriversNotRun: + @mock.patch(MPATH + "HAS_DEBCONF", True) + @mock.patch(M_TMP_PATH) + @mock.patch(MPATH + "install_drivers") + def test_no_cfg_drivers_does_nothing( + self, + m_install_drivers, + m_tmp, + m_debconf, + tmpdir, + ): + m_tmp.return_value = tmpdir + m_log = mock.MagicMock() + myCloud = mock.MagicMock() + version_none_cfg = {} + drivers.handle( + "ubuntu_drivers", version_none_cfg, myCloud, m_log, None + ) + assert 0 == m_install_drivers.call_count + assert ( + mock.call( + "Skipping module named %s, no 'drivers' key in config", + "ubuntu_drivers", + ) + == m_log.debug.call_args_list[-1] ) - def test_specifying_a_version_doesnt_override_license_acceptance(self): - self._assert_inert_with_config( - { - "drivers": { - "nvidia": {"license-accepted": False, "version": "123"} - } - } + @mock.patch(MPATH + "HAS_DEBCONF", False) + @mock.patch(M_TMP_PATH) + @mock.patch(MPATH + "install_drivers") + def test_has_not_debconf_does_nothing( + self, + m_install_drivers, + m_tmp, + m_debconf, + tmpdir, + ): + m_tmp.return_value = tmpdir + m_log = mock.MagicMock() + myCloud = mock.MagicMock() + version_none_cfg = {"drivers": {"nvidia": {"license-accepted": True}}} + drivers.handle( + "ubuntu_drivers", version_none_cfg, myCloud, m_log, None + ) + assert 0 == m_install_drivers.call_count + assert ( + mock.call( + "Skipping module named %s, 'python3-debconf' is not installed", + "ubuntu_drivers", + ) + == m_log.warning.call_args_list[-1] ) diff --git a/tests/unittests/distros/test_sysconfig.py b/tests/unittests/distros/test_sysconfig.py index d0979e17..9c3a2018 100644 --- a/tests/unittests/distros/test_sysconfig.py +++ b/tests/unittests/distros/test_sysconfig.py @@ -65,9 +65,7 @@ USEMD5=no""" conf["IPV6TO4_ROUTING"] = "blah \tblah" contents2 = str(conf).strip() # Should be requoted due to whitespace - self.assertRegMatches( - contents2, r"IPV6TO4_ROUTING=[\']blah\s+blah[\']" - ) + self.assertRegex(contents2, r"IPV6TO4_ROUTING=[\']blah\s+blah[\']") def test_parse_no_adjust_shell(self): conf = SysConf("".splitlines()) diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index 7846d0d3..e859aba4 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -110,14 +110,14 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): def test_no_arguments_shows_error_message(self): exit_code = self._call_main() - missing_subcommand_message = [ - "too few arguments", # python2.7 msg - "the following arguments are required: subcommand", # python3 msg - ] + missing_subcommand_message = ( + "the following arguments are required: subcommand" + ) error = self.stderr.getvalue() - matches = [msg in error for msg in missing_subcommand_message] - self.assertTrue( - any(matches), "Did not find error message for missing subcommand" + self.assertIn( + missing_subcommand_message, + error, + "Did not find error message for missing subcommand", ) self.assertEqual(2, exit_code) diff --git a/tests/unittests/test_net.py b/tests/unittests/test_net.py index cf7c9e24..bfc13734 100644 --- a/tests/unittests/test_net.py +++ b/tests/unittests/test_net.py @@ -443,7 +443,7 @@ network: macaddress: 68:05:ca:64:d3:6c mtu: 9000 parameters: - gratuitious-arp: 1 + gratuitous-arp: 1 bond1: interfaces: - ens4 @@ -2987,7 +2987,7 @@ pre-down route del -net 10.0.0.0/8 gw 11.0.0.1 metric 3 || true parameters: down-delay: 10 fail-over-mac-policy: active - gratuitious-arp: 5 + gratuitous-arp: 5 mii-monitor-interval: 100 mode: active-backup primary: bond0s0 @@ -3095,7 +3095,7 @@ iface bond0 inet6 static parameters: down-delay: 10 fail-over-mac-policy: active - gratuitious-arp: 5 + gratuitous-arp: 5 mii-monitor-interval: 100 mode: active-backup primary: bond0s0 @@ -3128,7 +3128,7 @@ iface bond0 inet6 static parameters: down-delay: 10 fail-over-mac-policy: active - gratuitious-arp: 5 + gratuitous-arp: 5 mii-monitor-interval: 100 mode: active-backup primary: bond0s0 @@ -6782,7 +6782,7 @@ class TestNetplanRoundTrip(CiTestCase): entry = { "yaml": NETPLAN_BOND_GRAT_ARP, "expected_netplan": NETPLAN_BOND_GRAT_ARP.replace( - "gratuitous", "gratuitious" + "gratuitious", "gratuitous" ), } network_config = yaml.load(entry["yaml"]).get("network") diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index ac898e3e..28cab205 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -754,9 +754,7 @@ class TestUdevadmSettle(CiTestCase): @mock.patch("os.path.exists") class TestGetLinuxDistro(CiTestCase): def setUp(self): - # python2 has no lru_cache, and therefore, no cache_clear() - if hasattr(util.get_linux_distro, "cache_clear"): - util.get_linux_distro.cache_clear() + util.get_linux_distro.cache_clear() @classmethod def os_release_exists(self, path): diff --git a/tools/read-version b/tools/read-version index 02c90643..c5cd153f 100755 --- a/tools/read-version +++ b/tools/read-version @@ -11,19 +11,11 @@ if "avoid-pep8-E402-import-not-top-of-file": from cloudinit import version as ci_version -def tiny_p(cmd, capture=True): - # python 2.6 doesn't have check_output - stdout = subprocess.PIPE +def tiny_p(cmd): stderr = subprocess.PIPE - sp = subprocess.Popen(cmd, stdout=stdout, - stderr=stderr, stdin=None, - universal_newlines=True) - (out, err) = sp.communicate() - ret = sp.returncode - if ret not in [0]: - raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" % - (cmd, ret, out, err)) - return out + return subprocess.check_output( + cmd, stderr=stderr, stdin=None, universal_newlines=True + ) def which(program): |