diff options
Diffstat (limited to 'cloudinit/sources/DataSourceAzure.py')
| -rwxr-xr-x | cloudinit/sources/DataSourceAzure.py | 1350 |
1 files changed, 762 insertions, 588 deletions
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index eee98fa8..a8b403e8 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -5,66 +5,62 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 -from collections import namedtuple import crypt -from functools import partial import os import os.path import re -from time import time -from time import sleep -from xml.dom import minidom import xml.etree.ElementTree as ET +from collections import namedtuple from enum import Enum +from functools import partial +from time import sleep, time +from xml.dom import minidom + import requests from cloudinit import dmi from cloudinit import log as logging -from cloudinit import net +from cloudinit import net, sources, ssh_util, subp, util from cloudinit.event import EventScope, EventType from cloudinit.net import device_driver from cloudinit.net.dhcp import EphemeralDHCPv4 -from cloudinit import sources -from cloudinit.sources.helpers import netlink -from cloudinit import ssh_util -from cloudinit import subp -from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc -from cloudinit import util from cloudinit.reporting import events - +from cloudinit.sources.helpers import netlink from cloudinit.sources.helpers.azure import ( DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE, + EphemeralDHCPv4WithReporting, azure_ds_reporter, azure_ds_telemetry_reporter, - get_metadata_from_fabric, + build_minimal_ovf, + dhcp_log_cb, get_boot_telemetry, + get_metadata_from_fabric, get_system_info, - report_diagnostic_event, - EphemeralDHCPv4WithReporting, is_byte_swapped, - dhcp_log_cb, push_log_to_kvp, + report_diagnostic_event, report_failure_to_fabric, - build_minimal_ovf) +) +from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc LOG = logging.getLogger(__name__) -DS_NAME = 'Azure' +DS_NAME = "Azure" DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} # azure systems will always have a resource disk, and 66-azure-ephemeral.rules # ensures that it gets linked to this path. -RESOURCE_DISK_PATH = '/dev/disk/cloud/azure_resource' -LEASE_FILE = '/var/lib/dhcp/dhclient.eth0.leases' -DEFAULT_FS = 'ext4' +RESOURCE_DISK_PATH = "/dev/disk/cloud/azure_resource" +LEASE_FILE = "/var/lib/dhcp/dhclient.eth0.leases" +DEFAULT_FS = "ext4" # DMI chassis-asset-tag is set static for all azure instances -AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77' +AZURE_CHASSIS_ASSET_TAG = "7783-7084-3265-9085-8269-3286-77" REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds" REPROVISION_NIC_ATTACH_MARKER_FILE = "/var/lib/cloud/data/wait_for_nic_attach" REPROVISION_NIC_DETACHED_MARKER_FILE = "/var/lib/cloud/data/nic_detached" REPORTED_READY_MARKER_FILE = "/var/lib/cloud/data/reported_ready" -AGENT_SEED_DIR = '/var/lib/waagent' -DEFAULT_PROVISIONING_ISO_DEV = '/dev/sr0' +AGENT_SEED_DIR = "/var/lib/waagent" +DEFAULT_PROVISIONING_ISO_DEV = "/dev/sr0" # In the event where the IMDS primary server is not # available, it takes 1s to fallback to the secondary one @@ -90,10 +86,10 @@ PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0" # List of static scripts and network config artifacts created by # stock ubuntu suported images. UBUNTU_EXTENDED_NETWORK_SCRIPTS = [ - '/etc/netplan/90-hotplug-azure.yaml', - '/usr/local/sbin/ephemeral_eth.sh', - '/etc/udev/rules.d/10-net-device-added.rules', - '/run/network/interfaces.ephemeral.d', + "/etc/netplan/90-hotplug-azure.yaml", + "/usr/local/sbin/ephemeral_eth.sh", + "/etc/udev/rules.d/10-net-device-added.rules", + "/run/network/interfaces.ephemeral.d", ] # This list is used to blacklist devices that will be considered @@ -113,7 +109,7 @@ UBUNTU_EXTENDED_NETWORK_SCRIPTS = [ # https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series # https://docs.microsoft.com/en-us/azure/virtual-machines/dv3-dsv3-series # https://docs.microsoft.com/en-us/azure/virtual-machines/ev3-esv3-series -BLACKLIST_DRIVERS = ['mlx4_core', 'mlx5_core'] +BLACKLIST_DRIVERS = ["mlx4_core", "mlx5_core"] def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid): @@ -127,11 +123,13 @@ def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid): if re.search(r"pnpinfo", line): fields = line.split() if len(fields) >= 3: - columns = fields[2].split('=') - if (len(columns) >= 2 and - columns[0] == "deviceid" and - columns[1].startswith(deviceid)): - comps = fields[0].split('.') + columns = fields[2].split("=") + if ( + len(columns) >= 2 + and columns[0] == "deviceid" + and columns[1].startswith(deviceid) + ): + comps = fields[0].split(".") return comps[2] return None @@ -165,9 +163,9 @@ def find_dev_from_busdev(camcontrol_out, busdev): """ for line in camcontrol_out.splitlines(): if re.search(busdev, line): - items = line.split('(') + items = line.split("(") if len(items) == 2: - dev_pass = items[1].split(',') + dev_pass = items[1].split(",") return dev_pass[0] return None @@ -176,7 +174,7 @@ def execute_or_debug(cmd, fail_ret=None): try: return subp.subp(cmd)[0] except subp.ProcessExecutionError: - LOG.debug("Failed to execute: %s", ' '.join(cmd)) + LOG.debug("Failed to execute: %s", " ".join(cmd)) return fail_ret @@ -185,11 +183,11 @@ def get_dev_storvsc_sysctl(): def get_camcontrol_dev_bus(): - return execute_or_debug(['camcontrol', 'devlist', '-b']) + return execute_or_debug(["camcontrol", "devlist", "-b"]) def get_camcontrol_dev(): - return execute_or_debug(['camcontrol', 'devlist']) + return execute_or_debug(["camcontrol", "devlist"]) def get_resource_disk_on_freebsd(port_id): @@ -236,8 +234,8 @@ def get_resource_disk_on_freebsd(port_id): # update the FreeBSD specific information if util.is_FreeBSD(): - LEASE_FILE = '/var/db/dhclient.leases.hn0' - DEFAULT_FS = 'freebsd-ufs' + LEASE_FILE = "/var/db/dhclient.leases.hn0" + DEFAULT_FS = "freebsd-ufs" res_disk = get_resource_disk_on_freebsd(1) if res_disk is not None: LOG.debug("resource disk is not None") @@ -248,52 +246,55 @@ if util.is_FreeBSD(): PLATFORM_ENTROPY_SOURCE = None BUILTIN_DS_CONFIG = { - 'data_dir': AGENT_SEED_DIR, - 'disk_aliases': {'ephemeral0': RESOURCE_DISK_PATH}, - 'dhclient_lease_file': LEASE_FILE, - 'apply_network_config': True, # Use IMDS published network configuration + "data_dir": AGENT_SEED_DIR, + "disk_aliases": {"ephemeral0": RESOURCE_DISK_PATH}, + "dhclient_lease_file": LEASE_FILE, + "apply_network_config": True, # Use IMDS published network configuration } # RELEASE_BLOCKER: Xenial and earlier apply_network_config default is False BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG = { - 'disk_setup': { - 'ephemeral0': {'table_type': 'gpt', - 'layout': [100], - 'overwrite': True}, + "disk_setup": { + "ephemeral0": { + "table_type": "gpt", + "layout": [100], + "overwrite": True, + }, }, - 'fs_setup': [{'filesystem': DEFAULT_FS, - 'device': 'ephemeral0.1'}], + "fs_setup": [{"filesystem": DEFAULT_FS, "device": "ephemeral0.1"}], } -DS_CFG_PATH = ['datasource', DS_NAME] -DS_CFG_KEY_PRESERVE_NTFS = 'never_destroy_ntfs' -DEF_EPHEMERAL_LABEL = 'Temporary Storage' +DS_CFG_PATH = ["datasource", DS_NAME] +DS_CFG_KEY_PRESERVE_NTFS = "never_destroy_ntfs" +DEF_EPHEMERAL_LABEL = "Temporary Storage" # The redacted password fails to meet password complexity requirements # so we can safely use this to mask/redact the password in the ovf-env.xml -DEF_PASSWD_REDACTION = 'REDACTED' +DEF_PASSWD_REDACTION = "REDACTED" class DataSourceAzure(sources.DataSource): - dsname = 'Azure' - default_update_events = {EventScope.NETWORK: { - EventType.BOOT_NEW_INSTANCE, - EventType.BOOT, - }} + dsname = "Azure" + default_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + } + } _negotiated = False _metadata_imds = sources.UNSET _ci_pkl_version = 1 def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.seed_dir = os.path.join(paths.seed_dir, 'azure') + self.seed_dir = os.path.join(paths.seed_dir, "azure") self.cfg = {} self.seed = None - self.ds_cfg = util.mergemanydict([ - util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), - BUILTIN_DS_CONFIG]) - self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file') + self.ds_cfg = util.mergemanydict( + [util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG] + ) + self.dhclient_lease_file = self.ds_cfg.get("dhclient_lease_file") self._network_config = None self._ephemeral_dhcp_ctx = None self.failed_desired_api_version = False @@ -312,13 +313,13 @@ class DataSourceAzure(sources.DataSource): def _get_subplatform(self): """Return the subplatform metadata source details.""" - if self.seed.startswith('/dev'): - subplatform_type = 'config-disk' - elif self.seed.lower() == 'imds': - subplatform_type = 'imds' + if self.seed.startswith("/dev"): + subplatform_type = "config-disk" + elif self.seed.lower() == "imds": + subplatform_type = "imds" else: - subplatform_type = 'seed-dir' - return '%s (%s)' % (subplatform_type, self.seed) + subplatform_type = "seed-dir" + return "%s (%s)" % (subplatform_type, self.seed) @azure_ds_telemetry_reporter def crawl_metadata(self): @@ -332,7 +333,7 @@ class DataSourceAzure(sources.DataSource): # azure removes/ejects the cdrom containing the ovf-env.xml # file on reboot. So, in order to successfully reboot we # need to look in the datadir and consider that valid - ddir = self.ds_cfg['data_dir'] + ddir = self.ds_cfg["data_dir"] # The order in which the candidates are inserted matters here, because # it determines the value of ret. More specifically, the first one in @@ -346,25 +347,28 @@ class DataSourceAzure(sources.DataSource): if os.path.isfile(REPROVISION_MARKER_FILE): reprovision = True metadata_source = "IMDS" - report_diagnostic_event("Reprovision marker file already present " - "before crawling Azure metadata: %s" % - REPROVISION_MARKER_FILE, - logger_func=LOG.debug) + report_diagnostic_event( + "Reprovision marker file already present " + "before crawling Azure metadata: %s" % REPROVISION_MARKER_FILE, + logger_func=LOG.debug, + ) elif os.path.isfile(REPROVISION_NIC_ATTACH_MARKER_FILE): reprovision_after_nic_attach = True metadata_source = "NIC_ATTACH_MARKER_PRESENT" - report_diagnostic_event("Reprovision nic attach marker file " - "already present before crawling Azure " - "metadata: %s" % - REPROVISION_NIC_ATTACH_MARKER_FILE, - logger_func=LOG.debug) + report_diagnostic_event( + "Reprovision nic attach marker file " + "already present before crawling Azure " + "metadata: %s" % REPROVISION_NIC_ATTACH_MARKER_FILE, + logger_func=LOG.debug, + ) else: for src in list_possible_azure_ds(self.seed_dir, ddir): try: if src.startswith("/dev/"): if util.is_FreeBSD(): - ret = util.mount_cb(src, load_azure_ds_dir, - mtype="udf") + ret = util.mount_cb( + src, load_azure_ds_dir, mtype="udf" + ) else: ret = util.mount_cb(src, load_azure_ds_dir) # save the device for ejection later @@ -377,36 +381,33 @@ class DataSourceAzure(sources.DataSource): except NonAzureDataSource: report_diagnostic_event( "Did not find Azure data source in %s" % src, - logger_func=LOG.debug) + logger_func=LOG.debug, + ) continue except util.MountFailedError: report_diagnostic_event( - '%s was not mountable' % src, - logger_func=LOG.debug) + "%s was not mountable" % src, logger_func=LOG.debug + ) ovf_is_accessible = False - empty_md = {'local-hostname': ''} + empty_md = {"local-hostname": ""} empty_cfg = dict( - system_info=dict( - default_user=dict( - name='' - ) - ) + system_info=dict(default_user=dict(name="")) ) - ret = (empty_md, '', empty_cfg, {}) - metadata_source = 'IMDS' + ret = (empty_md, "", empty_cfg, {}) + metadata_source = "IMDS" continue except BrokenAzureDataSource as exc: - msg = 'BrokenAzureDataSource: %s' % exc + msg = "BrokenAzureDataSource: %s" % exc report_diagnostic_event(msg, logger_func=LOG.error) raise sources.InvalidMetaDataException(msg) report_diagnostic_event( "Found provisioning metadata in %s" % metadata_source, - logger_func=LOG.debug) + logger_func=LOG.debug, + ) imds_md = self.get_imds_data_with_api_fallback( - self.fallback_interface, - retries=10 + self.fallback_interface, retries=10 ) # reset _fallback_interface so that if the code enters reprovisioning @@ -414,16 +415,17 @@ class DataSourceAzure(sources.DataSource): self._fallback_interface = None if not imds_md and not ovf_is_accessible: - msg = 'No OVF or IMDS available' + msg = "No OVF or IMDS available" report_diagnostic_event(msg) raise sources.InvalidMetaDataException(msg) - perform_reprovision = ( - reprovision or - self._should_reprovision(ret, imds_md)) + perform_reprovision = reprovision or self._should_reprovision( + ret, imds_md + ) perform_reprovision_after_nic_attach = ( - reprovision_after_nic_attach or - self._should_reprovision_after_nic_attach(ret, imds_md)) + reprovision_after_nic_attach + or self._should_reprovision_after_nic_attach(ret, imds_md) + ) if perform_reprovision or perform_reprovision_after_nic_attach: if util.is_FreeBSD(): @@ -435,45 +437,50 @@ class DataSourceAzure(sources.DataSource): ret = self._reprovision() # fetch metadata again as it has changed after reprovisioning imds_md = self.get_imds_data_with_api_fallback( - self.fallback_interface, - retries=10 + self.fallback_interface, retries=10 ) (md, userdata_raw, cfg, files) = ret self.seed = metadata_source - crawled_data.update({ - 'cfg': cfg, - 'files': files, - 'metadata': util.mergemanydict( - [md, {'imds': imds_md}]), - 'userdata_raw': userdata_raw}) + crawled_data.update( + { + "cfg": cfg, + "files": files, + "metadata": util.mergemanydict([md, {"imds": imds_md}]), + "userdata_raw": userdata_raw, + } + ) imds_username = _username_from_imds(imds_md) imds_hostname = _hostname_from_imds(imds_md) imds_disable_password = _disable_password_from_imds(imds_md) if imds_username: - LOG.debug('Username retrieved from IMDS: %s', imds_username) - cfg['system_info']['default_user']['name'] = imds_username + LOG.debug("Username retrieved from IMDS: %s", imds_username) + cfg["system_info"]["default_user"]["name"] = imds_username if imds_hostname: - LOG.debug('Hostname retrieved from IMDS: %s', imds_hostname) - crawled_data['metadata']['local-hostname'] = imds_hostname + LOG.debug("Hostname retrieved from IMDS: %s", imds_hostname) + crawled_data["metadata"]["local-hostname"] = imds_hostname if imds_disable_password: LOG.debug( - 'Disable password retrieved from IMDS: %s', - imds_disable_password + "Disable password retrieved from IMDS: %s", + imds_disable_password, ) - crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501 + crawled_data["metadata"][ + "disable_password" + ] = imds_disable_password - if metadata_source == 'IMDS' and not crawled_data['files']: + if metadata_source == "IMDS" and not crawled_data["files"]: try: contents = build_minimal_ovf( username=imds_username, hostname=imds_hostname, - disableSshPwd=imds_disable_password) - crawled_data['files'] = {'ovf-env.xml': contents} + disableSshPwd=imds_disable_password, + ) + crawled_data["files"] = {"ovf-env.xml": contents} except Exception as e: report_diagnostic_event( "Failed to construct OVF from IMDS data %s" % e, - logger_func=LOG.debug) + logger_func=LOG.debug, + ) # only use userdata from imds if OVF did not provide custom data # userdata provided by IMDS is always base64 encoded @@ -482,48 +489,53 @@ class DataSourceAzure(sources.DataSource): if imds_userdata: LOG.debug("Retrieved userdata from IMDS") try: - crawled_data['userdata_raw'] = base64.b64decode( - ''.join(imds_userdata.split())) + crawled_data["userdata_raw"] = base64.b64decode( + "".join(imds_userdata.split()) + ) except Exception: report_diagnostic_event( - "Bad userdata in IMDS", - logger_func=LOG.warning) + "Bad userdata in IMDS", logger_func=LOG.warning + ) if not metadata_source: - msg = 'No Azure metadata found' + msg = "No Azure metadata found" report_diagnostic_event(msg, logger_func=LOG.error) raise sources.InvalidMetaDataException(msg) else: report_diagnostic_event( - 'found datasource in %s' % metadata_source, - logger_func=LOG.debug) + "found datasource in %s" % metadata_source, + logger_func=LOG.debug, + ) if metadata_source == ddir: report_diagnostic_event( - "using files cached in %s" % ddir, logger_func=LOG.debug) + "using files cached in %s" % ddir, logger_func=LOG.debug + ) seed = _get_random_seed() if seed: - crawled_data['metadata']['random_seed'] = seed - crawled_data['metadata']['instance-id'] = self._iid() + crawled_data["metadata"]["random_seed"] = seed + crawled_data["metadata"]["instance-id"] = self._iid() if perform_reprovision or perform_reprovision_after_nic_attach: LOG.info("Reporting ready to Azure after getting ReprovisionData") - use_cached_ephemeral = ( - self.distro.networking.is_up(self.fallback_interface) and - getattr(self, '_ephemeral_dhcp_ctx', None)) + use_cached_ephemeral = self.distro.networking.is_up( + self.fallback_interface + ) and getattr(self, "_ephemeral_dhcp_ctx", None) if use_cached_ephemeral: self._report_ready(lease=self._ephemeral_dhcp_ctx.lease) self._ephemeral_dhcp_ctx.clean_network() # Teardown ephemeral else: try: with EphemeralDHCPv4WithReporting( - azure_ds_reporter) as lease: + azure_ds_reporter + ) as lease: self._report_ready(lease=lease) except Exception as e: report_diagnostic_event( "exception while reporting ready: %s" % e, - logger_func=LOG.error) + logger_func=LOG.error, + ) raise return crawled_data @@ -559,19 +571,24 @@ class DataSourceAzure(sources.DataSource): try: crawled_data = util.log_time( - logfunc=LOG.debug, msg='Crawl of metadata service', - func=self.crawl_metadata + logfunc=LOG.debug, + msg="Crawl of metadata service", + func=self.crawl_metadata, ) except Exception as e: report_diagnostic_event( - 'Could not crawl Azure metadata: %s' % e, - logger_func=LOG.error) + "Could not crawl Azure metadata: %s" % e, logger_func=LOG.error + ) self._report_failure( - description=DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE) + description=DEFAULT_REPORT_FAILURE_USER_VISIBLE_MESSAGE + ) return False - if (self.distro and self.distro.name == 'ubuntu' and - self.ds_cfg.get('apply_network_config')): + if ( + self.distro + and self.distro.name == "ubuntu" + and self.ds_cfg.get("apply_network_config") + ): maybe_remove_ubuntu_network_config_scripts() # Process crawled data and augment with various config defaults @@ -584,21 +601,25 @@ class DataSourceAzure(sources.DataSource): "Ephemeral resource disk '%s' exists. " "Merging default Azure cloud ephemeral disk configs." % devpath, - logger_func=LOG.debug) + logger_func=LOG.debug, + ) self.cfg = util.mergemanydict( - [crawled_data['cfg'], BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG]) + [crawled_data["cfg"], BUILTIN_CLOUD_EPHEMERAL_DISK_CONFIG] + ) else: report_diagnostic_event( "Ephemeral resource disk '%s' does not exist. " "Not merging default Azure cloud ephemeral disk configs." % devpath, - logger_func=LOG.debug) - self.cfg = crawled_data['cfg'] + logger_func=LOG.debug, + ) + self.cfg = crawled_data["cfg"] - self._metadata_imds = crawled_data['metadata']['imds'] + self._metadata_imds = crawled_data["metadata"]["imds"] self.metadata = util.mergemanydict( - [crawled_data['metadata'], DEFAULT_METADATA]) - self.userdata_raw = crawled_data['userdata_raw'] + [crawled_data["metadata"], DEFAULT_METADATA] + ) + self.userdata_raw = crawled_data["userdata_raw"] user_ds_cfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) self.ds_cfg = util.mergemanydict([user_ds_cfg, self.ds_cfg]) @@ -606,17 +627,19 @@ class DataSourceAzure(sources.DataSource): # walinux agent writes files world readable, but expects # the directory to be protected. write_files( - self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700) + self.ds_cfg["data_dir"], crawled_data["files"], dirmode=0o700 + ) return True @azure_ds_telemetry_reporter def get_imds_data_with_api_fallback( - self, - fallback_nic, - retries, - md_type=metadata_type.all, - exc_cb=retry_on_url_exc, - infinite=False): + self, + fallback_nic, + retries, + md_type=metadata_type.all, + exc_cb=retry_on_url_exc, + infinite=False, + ): """ Wrapper for get_metadata_from_imds so that we can have flexibility in which IMDS api-version we use. If a particular instance of IMDS @@ -628,30 +651,23 @@ class DataSourceAzure(sources.DataSource): if not self.failed_desired_api_version: for _ in range(retries): try: - LOG.info( - "Attempting IMDS api-version: %s", - IMDS_VER_WANT - ) + LOG.info("Attempting IMDS api-version: %s", IMDS_VER_WANT) return get_metadata_from_imds( fallback_nic=fallback_nic, retries=0, md_type=md_type, api_version=IMDS_VER_WANT, - exc_cb=exc_cb + exc_cb=exc_cb, ) except UrlError as err: LOG.info( - "UrlError with IMDS api-version: %s", - IMDS_VER_WANT + "UrlError with IMDS api-version: %s", IMDS_VER_WANT ) if err.code == 400: log_msg = "Fall back to IMDS api-version: {}".format( IMDS_VER_MIN ) - report_diagnostic_event( - log_msg, - logger_func=LOG.info - ) + report_diagnostic_event(log_msg, logger_func=LOG.info) self.failed_desired_api_version = True break @@ -662,11 +678,11 @@ class DataSourceAzure(sources.DataSource): md_type=md_type, api_version=IMDS_VER_MIN, exc_cb=exc_cb, - infinite=infinite + infinite=infinite, ) def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) + return self.ds_cfg["disk_aliases"].get(name) @azure_ds_telemetry_reporter def get_public_ssh_keys(self): @@ -687,15 +703,16 @@ class DataSourceAzure(sources.DataSource): OVF as a second option for environments that don't have IMDS. """ - LOG.debug('Retrieving public SSH keys') + LOG.debug("Retrieving public SSH keys") ssh_keys = [] keys_from_imds = True - LOG.debug('Attempting to get SSH keys from IMDS') + LOG.debug("Attempting to get SSH keys from IMDS") try: ssh_keys = [ - public_key['keyData'] - for public_key - in self.metadata['imds']['compute']['publicKeys'] + public_key["keyData"] + for public_key in self.metadata["imds"]["compute"][ + "publicKeys" + ] ] for key in ssh_keys: if not _key_is_openssh_formatted(key=key): @@ -703,33 +720,28 @@ class DataSourceAzure(sources.DataSource): break if not keys_from_imds: - log_msg = 'Keys not in OpenSSH format, using OVF' + log_msg = "Keys not in OpenSSH format, using OVF" else: - log_msg = 'Retrieved {} keys from IMDS'.format( - len(ssh_keys) - if ssh_keys is not None - else 0 + log_msg = "Retrieved {} keys from IMDS".format( + len(ssh_keys) if ssh_keys is not None else 0 ) except KeyError: - log_msg = 'Unable to get keys from IMDS, falling back to OVF' + log_msg = "Unable to get keys from IMDS, falling back to OVF" keys_from_imds = False finally: report_diagnostic_event(log_msg, logger_func=LOG.debug) if not keys_from_imds: - LOG.debug('Attempting to get SSH keys from OVF') + LOG.debug("Attempting to get SSH keys from OVF") try: - ssh_keys = self.metadata['public-keys'] - log_msg = 'Retrieved {} keys from OVF'.format(len(ssh_keys)) + ssh_keys = self.metadata["public-keys"] + log_msg = "Retrieved {} keys from OVF".format(len(ssh_keys)) except KeyError: - log_msg = 'No keys available from OVF' + log_msg = "No keys available from OVF" finally: report_diagnostic_event(log_msg, logger_func=LOG.debug) - return SSHKeys( - keys_from_imds=keys_from_imds, - ssh_keys=ssh_keys - ) + return SSHKeys(keys_from_imds=keys_from_imds, ssh_keys=ssh_keys) def get_config_obj(self): return self.cfg @@ -740,12 +752,13 @@ class DataSourceAzure(sources.DataSource): def _iid(self, previous=None): prev_iid_path = os.path.join( - self.paths.get_cpath('data'), 'instance-id') + self.paths.get_cpath("data"), "instance-id" + ) # Older kernels than 4.15 will have UPPERCASE product_uuid. # We don't want Azure to react to an UPPER/lower difference as a new # instance id as it rewrites SSH host keys. # LP: #1835584 - iid = dmi.read_dmi_data('system-uuid').lower() + iid = dmi.read_dmi_data("system-uuid").lower() if os.path.exists(prev_iid_path): previous = util.load_file(prev_iid_path).strip() if previous.lower() == iid: @@ -759,22 +772,26 @@ class DataSourceAzure(sources.DataSource): @azure_ds_telemetry_reporter def setup(self, is_new_instance): if self._negotiated is False: - LOG.debug("negotiating for %s (new_instance=%s)", - self.get_instance_id(), is_new_instance) + LOG.debug( + "negotiating for %s (new_instance=%s)", + self.get_instance_id(), + is_new_instance, + ) fabric_data = self._negotiate() LOG.debug("negotiating returned %s", fabric_data) if fabric_data: self.metadata.update(fabric_data) self._negotiated = True else: - LOG.debug("negotiating already done for %s", - self.get_instance_id()) + LOG.debug( + "negotiating already done for %s", self.get_instance_id() + ) @azure_ds_telemetry_reporter def _wait_for_nic_detach(self, nl_sock): """Use the netlink socket provided to wait for nic detach event. - NOTE: The function doesn't close the socket. The caller owns closing - the socket and disposing it safely. + NOTE: The function doesn't close the socket. The caller owns closing + the socket and disposing it safely. """ try: ifname = None @@ -782,21 +799,27 @@ class DataSourceAzure(sources.DataSource): # Preprovisioned VM will only have one NIC, and it gets # detached immediately after deployment. with events.ReportEventStack( - name="wait-for-nic-detach", - description=("wait for nic detach"), - parent=azure_ds_reporter): + name="wait-for-nic-detach", + description="wait for nic detach", + parent=azure_ds_reporter, + ): ifname = netlink.wait_for_nic_detach_event(nl_sock) if ifname is None: - msg = ("Preprovisioned nic not detached as expected. " - "Proceeding without failing.") + msg = ( + "Preprovisioned nic not detached as expected. " + "Proceeding without failing." + ) report_diagnostic_event(msg, logger_func=LOG.warning) else: - report_diagnostic_event("The preprovisioned nic %s is detached" - % ifname, logger_func=LOG.warning) + report_diagnostic_event( + "The preprovisioned nic %s is detached" % ifname, + logger_func=LOG.warning, + ) path = REPROVISION_NIC_DETACHED_MARKER_FILE LOG.info("Creating a marker file for nic detached: %s", path) - util.write_file(path, "{pid}: {time}\n".format( - pid=os.getpid(), time=time())) + util.write_file( + path, "{pid}: {time}\n".format(pid=os.getpid(), time=time()) + ) except AssertionError as error: report_diagnostic_event(error, logger_func=LOG.error) raise @@ -804,14 +827,15 @@ class DataSourceAzure(sources.DataSource): @azure_ds_telemetry_reporter def wait_for_link_up(self, ifname): """In cases where the link state is still showing down after a nic is - hot-attached, we can attempt to bring it up by forcing the hv_netvsc - drivers to query the link state by unbinding and then binding the - device. This function attempts infinitely until the link is up, - because we cannot proceed further until we have a stable link.""" + hot-attached, we can attempt to bring it up by forcing the hv_netvsc + drivers to query the link state by unbinding and then binding the + device. This function attempts infinitely until the link is up, + because we cannot proceed further until we have a stable link.""" if self.distro.networking.try_set_link_up(ifname): - report_diagnostic_event("The link %s is already up." % ifname, - logger_func=LOG.info) + report_diagnostic_event( + "The link %s is already up." % ifname, logger_func=LOG.info + ) return LOG.debug("Attempting to bring %s up", ifname) @@ -820,22 +844,27 @@ class DataSourceAzure(sources.DataSource): LOG.info("Unbinding and binding the interface %s", ifname) while True: - devicename = net.read_sys_net(ifname, - 'device/device_id').strip('{}') - util.write_file('/sys/bus/vmbus/drivers/hv_netvsc/unbind', - devicename) - util.write_file('/sys/bus/vmbus/drivers/hv_netvsc/bind', - devicename) + devicename = net.read_sys_net(ifname, "device/device_id").strip( + "{}" + ) + util.write_file( + "/sys/bus/vmbus/drivers/hv_netvsc/unbind", devicename + ) + util.write_file( + "/sys/bus/vmbus/drivers/hv_netvsc/bind", devicename + ) attempts = attempts + 1 if self.distro.networking.try_set_link_up(ifname): - msg = "The link %s is up after %s attempts" % (ifname, - attempts) + msg = "The link %s is up after %s attempts" % ( + ifname, + attempts, + ) report_diagnostic_event(msg, logger_func=LOG.info) return if attempts % 10 == 0: - msg = ("Link is not up after %d attempts to rebind" % attempts) + msg = "Link is not up after %d attempts to rebind" % attempts report_diagnostic_event(msg, logger_func=LOG.info) LOG.info(msg) @@ -844,13 +873,17 @@ class DataSourceAzure(sources.DataSource): # again. sleep_duration = 0.5 max_status_polls = 20 - LOG.debug("Polling %d seconds for primary NIC link up after " - "rebind.", sleep_duration * max_status_polls) + LOG.debug( + "Polling %d seconds for primary NIC link up after rebind.", + sleep_duration * max_status_polls, + ) for i in range(0, max_status_polls): if self.distro.networking.is_up(ifname): - msg = ("After %d attempts to rebind, link is up after " - "polling the link status %d times" % (attempts, i)) + msg = ( + "After %d attempts to rebind, link is up after " + "polling the link status %d times" % (attempts, i) + ) report_diagnostic_event(msg, logger_func=LOG.info) LOG.debug(msg) return @@ -860,40 +893,47 @@ class DataSourceAzure(sources.DataSource): @azure_ds_telemetry_reporter def _create_report_ready_marker(self): path = REPORTED_READY_MARKER_FILE - LOG.info( - "Creating a marker file to report ready: %s", path) - util.write_file(path, "{pid}: {time}\n".format( - pid=os.getpid(), time=time())) + LOG.info("Creating a marker file to report ready: %s", path) + util.write_file( + path, "{pid}: {time}\n".format(pid=os.getpid(), time=time()) + ) report_diagnostic_event( - 'Successfully created reported ready marker file ' - 'while in the preprovisioning pool.', - logger_func=LOG.debug) + "Successfully created reported ready marker file " + "while in the preprovisioning pool.", + logger_func=LOG.debug, + ) @azure_ds_telemetry_reporter def _report_ready_if_needed(self): """Report ready to the platform if the marker file is not present, and create the marker file. """ - have_not_reported_ready = ( - not os.path.isfile(REPORTED_READY_MARKER_FILE)) + have_not_reported_ready = not os.path.isfile( + REPORTED_READY_MARKER_FILE + ) if have_not_reported_ready: - report_diagnostic_event("Reporting ready before nic detach", - logger_func=LOG.info) + report_diagnostic_event( + "Reporting ready before nic detach", logger_func=LOG.info + ) try: with EphemeralDHCPv4WithReporting(azure_ds_reporter) as lease: self._report_ready(lease=lease) except Exception as e: - report_diagnostic_event("Exception reporting ready during " - "preprovisioning before nic detach: %s" - % e, logger_func=LOG.error) + report_diagnostic_event( + "Exception reporting ready during " + "preprovisioning before nic detach: %s" % e, + logger_func=LOG.error, + ) raise self._create_report_ready_marker() else: - report_diagnostic_event("Already reported ready before nic detach." - " The marker file already exists: %s" % - REPORTED_READY_MARKER_FILE, - logger_func=LOG.error) + report_diagnostic_event( + "Already reported ready before nic detach." + " The marker file already exists: %s" + % REPORTED_READY_MARKER_FILE, + logger_func=LOG.error, + ) @azure_ds_telemetry_reporter def _check_if_nic_is_primary(self, ifname): @@ -915,20 +955,26 @@ class DataSourceAzure(sources.DataSource): # the primary NIC. try: with events.ReportEventStack( - name="obtain-dhcp-lease", - description=("obtain dhcp lease for %s when attempting to " - "determine primary NIC during reprovision of " - "a pre-provisioned VM" % ifname), - parent=azure_ds_reporter): + name="obtain-dhcp-lease", + description=( + "obtain dhcp lease for %s when attempting to " + "determine primary NIC during reprovision of " + "a pre-provisioned VM" + ) + % ifname, + parent=azure_ds_reporter, + ): dhcp_ctx = EphemeralDHCPv4( - iface=ifname, - dhcp_log_func=dhcp_log_cb) + iface=ifname, dhcp_log_func=dhcp_log_cb + ) dhcp_ctx.obtain_lease() except Exception as e: - report_diagnostic_event("Giving up. Failed to obtain dhcp lease " - "for %s when attempting to determine " - "primary NIC during reprovision due to %s" - % (ifname, e), logger_func=LOG.error) + report_diagnostic_event( + "Giving up. Failed to obtain dhcp lease " + "for %s when attempting to determine " + "primary NIC during reprovision due to %s" % (ifname, e), + logger_func=LOG.error, + ) raise # Retry polling network metadata for a limited duration only when the @@ -953,13 +999,15 @@ class DataSourceAzure(sources.DataSource): report_diagnostic_event( "Ran into exception when attempting to reach %s " "after %d polls." % (msg, metadata_poll_count), - logger_func=LOG.error) + logger_func=LOG.error, + ) if isinstance(exc, UrlError): - report_diagnostic_event("poll IMDS with %s failed. " - "Exception: %s and code: %s" % - (msg, exc.cause, exc.code), - logger_func=LOG.error) + report_diagnostic_event( + "poll IMDS with %s failed. Exception: %s and code: %s" + % (msg, exc.cause, exc.code), + logger_func=LOG.error, + ) # Retry up to a certain limit for both timeout and network # unreachable errors. @@ -967,7 +1015,7 @@ class DataSourceAzure(sources.DataSource): exc.cause, (requests.Timeout, requests.ConnectionError) ): expected_errors_count = expected_errors_count + 1 - return (expected_errors_count <= 10) + return expected_errors_count <= 10 return True # Primary nic detection will be optimized in the future. The fact that @@ -975,17 +1023,16 @@ class DataSourceAzure(sources.DataSource): # could add several seconds of delay. try: imds_md = self.get_imds_data_with_api_fallback( - ifname, - 0, - metadata_type.network, - network_metadata_exc_cb, - True + ifname, 0, metadata_type.network, network_metadata_exc_cb, True ) except Exception as e: LOG.warning( "Failed to get network metadata using nic %s. Attempt to " "contact IMDS failed with error %s. Assuming this is not the " - "primary nic.", ifname, e) + "primary nic.", + ifname, + e, + ) finally: # If we are not the primary nic, then clean the dhcp context. if imds_md is None: @@ -1000,10 +1047,11 @@ class DataSourceAzure(sources.DataSource): self._ephemeral_dhcp_ctx = dhcp_ctx # Set the expected nic count based on the response received. - expected_nic_count = len( - imds_md['interface']) - report_diagnostic_event("Expected nic count: %d" % - expected_nic_count, logger_func=LOG.info) + expected_nic_count = len(imds_md["interface"]) + report_diagnostic_event( + "Expected nic count: %d" % expected_nic_count, + logger_func=LOG.info, + ) return is_primary, expected_nic_count @@ -1028,17 +1076,22 @@ class DataSourceAzure(sources.DataSource): while True: ifname = None with events.ReportEventStack( - name="wait-for-nic-attach", - description=("wait for nic attach after %d nics have " - "been attached" % len(nics_found)), - parent=azure_ds_reporter): - ifname = netlink.wait_for_nic_attach_event(nl_sock, - nics_found) + name="wait-for-nic-attach", + description=( + "wait for nic attach after %d nics have been attached" + % len(nics_found) + ), + parent=azure_ds_reporter, + ): + ifname = netlink.wait_for_nic_attach_event( + nl_sock, nics_found + ) # wait_for_nic_attach_event guarantees that ifname it not None nics_found.append(ifname) - report_diagnostic_event("Detected nic %s attached." % ifname, - logger_func=LOG.info) + report_diagnostic_event( + "Detected nic %s attached." % ifname, logger_func=LOG.info + ) # Attempt to bring the interface's operating state to # UP in case it is not already. @@ -1048,14 +1101,17 @@ class DataSourceAzure(sources.DataSource): # platform will attach the primary nic first so we # won't be in primary_nic_found = false state for long. if not primary_nic_found: - LOG.info("Checking if %s is the primary nic", - ifname) - (primary_nic_found, expected_nic_count) = ( - self._check_if_nic_is_primary(ifname)) + LOG.info("Checking if %s is the primary nic", ifname) + ( + primary_nic_found, + expected_nic_count, + ) = self._check_if_nic_is_primary(ifname) # Exit criteria: check if we've discovered all nics - if (expected_nic_count != -1 - and len(nics_found) >= expected_nic_count): + if ( + expected_nic_count != -1 + and len(nics_found) >= expected_nic_count + ): LOG.info("Found all the nics for this VM.") break @@ -1065,9 +1121,9 @@ class DataSourceAzure(sources.DataSource): @azure_ds_telemetry_reporter def _wait_for_all_nics_ready(self): """Wait for nic(s) to be hot-attached. There may be multiple nics - depending on the customer request. - But only primary nic would be able to communicate with wireserver - and IMDS. So we detect and save the primary nic to be used later. + depending on the customer request. + But only primary nic would be able to communicate with wireserver + and IMDS. So we detect and save the primary nic to be used later. """ nl_sock = None @@ -1075,7 +1131,8 @@ class DataSourceAzure(sources.DataSource): nl_sock = netlink.create_bound_netlink_socket() report_ready_marker_present = bool( - os.path.isfile(REPORTED_READY_MARKER_FILE)) + os.path.isfile(REPORTED_READY_MARKER_FILE) + ) # Report ready if the marker file is not already present. # The nic of the preprovisioned vm gets hot-detached as soon as @@ -1083,7 +1140,8 @@ class DataSourceAzure(sources.DataSource): self._report_ready_if_needed() has_nic_been_detached = bool( - os.path.isfile(REPROVISION_NIC_DETACHED_MARKER_FILE)) + os.path.isfile(REPROVISION_NIC_DETACHED_MARKER_FILE) + ) if not has_nic_been_detached: LOG.info("NIC has not been detached yet.") @@ -1097,12 +1155,14 @@ class DataSourceAzure(sources.DataSource): if not self.fallback_interface: self._wait_for_hot_attached_nics(nl_sock) else: - report_diagnostic_event("Skipping waiting for nic attach " - "because we already have a fallback " - "interface. Report Ready marker " - "present before detaching nics: %s" % - report_ready_marker_present, - logger_func=LOG.info) + report_diagnostic_event( + "Skipping waiting for nic attach " + "because we already have a fallback " + "interface. Report Ready marker " + "present before detaching nics: %s" + % report_ready_marker_present, + logger_func=LOG.info, + ) except netlink.NetlinkCreateSocketError as e: report_diagnostic_event(e, logger_func=LOG.warning) raise @@ -1115,8 +1175,7 @@ class DataSourceAzure(sources.DataSource): """Poll IMDS for the new provisioning data until we get a valid response. Then return the returned JSON object.""" url = "{}?api-version={}".format( - metadata_type.reprovisiondata.value, - IMDS_VER_MIN + metadata_type.reprovisiondata.value, IMDS_VER_MIN ) headers = {"Metadata": "true"} nl_sock = None @@ -1133,38 +1192,44 @@ class DataSourceAzure(sources.DataSource): if self.imds_poll_counter == self.imds_logging_threshold: # Reducing the logging frequency as we are polling IMDS self.imds_logging_threshold *= 2 - LOG.debug("Backing off logging threshold for the same " - "exception to %d", - self.imds_logging_threshold) - report_diagnostic_event("poll IMDS with %s failed. " - "Exception: %s and code: %s" % - (msg, exception.cause, - exception.code), - logger_func=LOG.debug) + LOG.debug( + "Backing off logging threshold for the same " + "exception to %d", + self.imds_logging_threshold, + ) + report_diagnostic_event( + "poll IMDS with %s failed. " + "Exception: %s and code: %s" + % (msg, exception.cause, exception.code), + logger_func=LOG.debug, + ) self.imds_poll_counter += 1 return True else: # If we get an exception while trying to call IMDS, we call # DHCP and setup the ephemeral network to acquire a new IP. - report_diagnostic_event("poll IMDS with %s failed. " - "Exception: %s and code: %s" % - (msg, exception.cause, - exception.code), - logger_func=LOG.warning) + report_diagnostic_event( + "poll IMDS with %s failed. Exception: %s and code: %s" + % (msg, exception.cause, exception.code), + logger_func=LOG.warning, + ) return False report_diagnostic_event( - "poll IMDS failed with an " - "unexpected exception: %s" % exception, - logger_func=LOG.warning) + "poll IMDS failed with an unexpected exception: %s" + % exception, + logger_func=LOG.warning, + ) return False # When the interface is hot-attached, we would have already # done dhcp and set the dhcp context. In that case, skip # the attempt to do dhcp. is_ephemeral_ctx_present = self._ephemeral_dhcp_ctx is not None - msg = ("Unexpected error. Dhcp context is not expected to be already " - "set when we need to wait for vnet switch") + msg = ( + "Unexpected error. Dhcp context is not expected to be already " + "set when we need to wait for vnet switch" + ) if is_ephemeral_ctx_present and report_ready: report_diagnostic_event(msg, logger_func=LOG.error) raise RuntimeError(msg) @@ -1178,11 +1243,13 @@ class DataSourceAzure(sources.DataSource): # Save our EphemeralDHCPv4 context to avoid repeated dhcp # later when we report ready with events.ReportEventStack( - name="obtain-dhcp-lease", - description="obtain dhcp lease", - parent=azure_ds_reporter): + name="obtain-dhcp-lease", + description="obtain dhcp lease", + parent=azure_ds_reporter, + ): self._ephemeral_dhcp_ctx = EphemeralDHCPv4( - dhcp_log_func=dhcp_log_cb) + dhcp_log_func=dhcp_log_cb + ) lease = self._ephemeral_dhcp_ctx.obtain_lease() if vnet_switched: @@ -1192,15 +1259,18 @@ class DataSourceAzure(sources.DataSource): nl_sock = netlink.create_bound_netlink_socket() except netlink.NetlinkCreateSocketError as e: report_diagnostic_event( - 'Failed to create bound netlink socket: %s' % e, - logger_func=LOG.warning) + "Failed to create bound netlink socket: %s" % e, + logger_func=LOG.warning, + ) self._ephemeral_dhcp_ctx.clean_network() break report_ready_succeeded = self._report_ready(lease=lease) if not report_ready_succeeded: - msg = ('Failed reporting ready while in ' - 'the preprovisioning pool.') + msg = ( + "Failed reporting ready while in " + "the preprovisioning pool." + ) report_diagnostic_event(msg, logger_func=LOG.error) self._ephemeral_dhcp_ctx.clean_network() raise sources.InvalidMetaDataException(msg) @@ -1210,31 +1280,37 @@ class DataSourceAzure(sources.DataSource): LOG.debug("Wait for vnetswitch to happen") with events.ReportEventStack( - name="wait-for-media-disconnect-connect", - description="wait for vnet switch", - parent=azure_ds_reporter): + name="wait-for-media-disconnect-connect", + description="wait for vnet switch", + parent=azure_ds_reporter, + ): try: netlink.wait_for_media_disconnect_connect( - nl_sock, lease['interface']) + nl_sock, lease["interface"] + ) except AssertionError as e: report_diagnostic_event( - 'Error while waiting for vnet switch: %s' % e, - logger_func=LOG.error) + "Error while waiting for vnet switch: %s" % e, + logger_func=LOG.error, + ) break vnet_switched = True self._ephemeral_dhcp_ctx.clean_network() else: with events.ReportEventStack( - name="get-reprovision-data-from-imds", - description="get reprovision data from imds", - parent=azure_ds_reporter): - return_val = readurl(url, - timeout=IMDS_TIMEOUT_IN_SECONDS, - headers=headers, - exception_cb=exc_cb, - infinite=True, - log_req_resp=False).contents + name="get-reprovision-data-from-imds", + description="get reprovision data from imds", + parent=azure_ds_reporter, + ): + return_val = readurl( + url, + timeout=IMDS_TIMEOUT_IN_SECONDS, + headers=headers, + exception_cb=exc_cb, + infinite=True, + log_req_resp=False, + ).contents break except UrlError: # Teardown our EphemeralDHCPv4 context on failure as we retry @@ -1248,12 +1324,14 @@ class DataSourceAzure(sources.DataSource): nl_sock.close() if vnet_switched: - report_diagnostic_event("attempted dhcp %d times after reuse" % - dhcp_attempts, - logger_func=LOG.debug) - report_diagnostic_event("polled imds %d times after reuse" % - self.imds_poll_counter, - logger_func=LOG.debug) + report_diagnostic_event( + "attempted dhcp %d times after reuse" % dhcp_attempts, + logger_func=LOG.debug, + ) + report_diagnostic_event( + "polled imds %d times after reuse" % self.imds_poll_counter, + logger_func=LOG.debug, + ) return return_val @@ -1264,52 +1342,63 @@ class DataSourceAzure(sources.DataSource): @param description: A description of the error encountered. @return: The success status of sending the failure signal. """ - unknown_245_key = 'unknown-245' + unknown_245_key = "unknown-245" try: - if (self.distro.networking.is_up(self.fallback_interface) and - getattr(self, '_ephemeral_dhcp_ctx', None) and - getattr(self._ephemeral_dhcp_ctx, 'lease', None) and - unknown_245_key in self._ephemeral_dhcp_ctx.lease): + if ( + self.distro.networking.is_up(self.fallback_interface) + and getattr(self, "_ephemeral_dhcp_ctx", None) + and getattr(self._ephemeral_dhcp_ctx, "lease", None) + and unknown_245_key in self._ephemeral_dhcp_ctx.lease + ): report_diagnostic_event( - 'Using cached ephemeral dhcp context ' - 'to report failure to Azure', logger_func=LOG.debug) + "Using cached ephemeral dhcp context " + "to report failure to Azure", + logger_func=LOG.debug, + ) report_failure_to_fabric( dhcp_opts=self._ephemeral_dhcp_ctx.lease[unknown_245_key], - description=description) + description=description, + ) self._ephemeral_dhcp_ctx.clean_network() # Teardown ephemeral return True except Exception as e: report_diagnostic_event( - 'Failed to report failure using ' - 'cached ephemeral dhcp context: %s' % e, - logger_func=LOG.error) + "Failed to report failure using " + "cached ephemeral dhcp context: %s" % e, + logger_func=LOG.error, + ) try: report_diagnostic_event( - 'Using new ephemeral dhcp to report failure to Azure', - logger_func=LOG.debug) + "Using new ephemeral dhcp to report failure to Azure", + logger_func=LOG.debug, + ) with EphemeralDHCPv4WithReporting(azure_ds_reporter) as lease: report_failure_to_fabric( - dhcp_opts=lease[unknown_245_key], - description=description) + dhcp_opts=lease[unknown_245_key], description=description + ) return True except Exception as e: report_diagnostic_event( - 'Failed to report failure using new ephemeral dhcp: %s' % e, - logger_func=LOG.debug) + "Failed to report failure using new ephemeral dhcp: %s" % e, + logger_func=LOG.debug, + ) try: report_diagnostic_event( - 'Using fallback lease to report failure to Azure') + "Using fallback lease to report failure to Azure" + ) report_failure_to_fabric( fallback_lease_file=self.dhclient_lease_file, - description=description) + description=description, + ) return True except Exception as e: report_diagnostic_event( - 'Failed to report failure using fallback lease: %s' % e, - logger_func=LOG.debug) + "Failed to report failure using fallback lease: %s" % e, + logger_func=LOG.debug, + ) return False @@ -1320,27 +1409,33 @@ class DataSourceAzure(sources.DataSource): @return: The success status of sending the ready signal. """ try: - get_metadata_from_fabric(fallback_lease_file=None, - dhcp_opts=lease['unknown-245'], - iso_dev=self.iso_dev) + get_metadata_from_fabric( + fallback_lease_file=None, + dhcp_opts=lease["unknown-245"], + iso_dev=self.iso_dev, + ) return True except Exception as e: report_diagnostic_event( "Error communicating with Azure fabric; You may experience " - "connectivity issues: %s" % e, logger_func=LOG.warning) + "connectivity issues: %s" % e, + logger_func=LOG.warning, + ) return False def _ppstype_from_imds(self, imds_md: dict = None) -> str: try: - return imds_md['extended']['compute']['ppsType'] + return imds_md["extended"]["compute"]["ppsType"] except Exception as e: report_diagnostic_event( - "Could not retrieve pps configuration from IMDS: %s" % - e, logger_func=LOG.debug) + "Could not retrieve pps configuration from IMDS: %s" % e, + logger_func=LOG.debug, + ) return None def _should_reprovision_after_nic_attach( - self, ovf_md, imds_md=None) -> bool: + self, ovf_md, imds_md=None + ) -> bool: """Whether or not we should wait for nic attach and then poll IMDS for reprovisioning data. Also sets a marker file to poll IMDS. @@ -1360,14 +1455,19 @@ class DataSourceAzure(sources.DataSource): return False (_md, _userdata_raw, cfg, _files) = ovf_md path = REPROVISION_NIC_ATTACH_MARKER_FILE - if (cfg.get('PreprovisionedVMType', None) == "Savable" or - self._ppstype_from_imds(imds_md) == "Savable" or - os.path.isfile(path)): + if ( + cfg.get("PreprovisionedVMType", None) == "Savable" + or self._ppstype_from_imds(imds_md) == "Savable" + or os.path.isfile(path) + ): if not os.path.isfile(path): - LOG.info("Creating a marker file to wait for nic attach: %s", - path) - util.write_file(path, "{pid}: {time}\n".format( - pid=os.getpid(), time=time())) + LOG.info( + "Creating a marker file to wait for nic attach: %s", path + ) + util.write_file( + path, + "{pid}: {time}\n".format(pid=os.getpid(), time=time()), + ) return True return False @@ -1386,15 +1486,18 @@ class DataSourceAzure(sources.DataSource): return False (_md, _userdata_raw, cfg, _files) = ovf_md path = REPROVISION_MARKER_FILE - if (cfg.get('PreprovisionedVm') is True or - cfg.get('PreprovisionedVMType', None) == 'Running' or - self._ppstype_from_imds(imds_md) == "Running" or - os.path.isfile(path)): + if ( + cfg.get("PreprovisionedVm") is True + or cfg.get("PreprovisionedVMType", None) == "Running" + or self._ppstype_from_imds(imds_md) == "Running" + or os.path.isfile(path) + ): if not os.path.isfile(path): - LOG.info("Creating a marker file to poll imds: %s", - path) - util.write_file(path, "{pid}: {time}\n".format( - pid=os.getpid(), time=time())) + LOG.info("Creating a marker file to poll imds: %s", path) + util.write_file( + path, + "{pid}: {time}\n".format(pid=os.getpid(), time=time()), + ) return True return False @@ -1402,35 +1505,35 @@ class DataSourceAzure(sources.DataSource): """Initiate the reprovisioning workflow.""" contents = self._poll_imds() with events.ReportEventStack( - name="reprovisioning-read-azure-ovf", - description="read azure ovf during reprovisioning", - parent=azure_ds_reporter): + name="reprovisioning-read-azure-ovf", + description="read azure ovf during reprovisioning", + parent=azure_ds_reporter, + ): md, ud, cfg = read_azure_ovf(contents) - return (md, ud, cfg, {'ovf-env.xml': contents}) + return (md, ud, cfg, {"ovf-env.xml": contents}) @azure_ds_telemetry_reporter def _negotiate(self): """Negotiate with fabric and return data from it. - On success, returns a dictionary including 'public_keys'. - On failure, returns False. + On success, returns a dictionary including 'public_keys'. + On failure, returns False. """ pubkey_info = None ssh_keys_and_source = self._get_public_ssh_keys_and_source() if not ssh_keys_and_source.keys_from_imds: - pubkey_info = self.cfg.get('_pubkeys', None) - log_msg = 'Retrieved {} fingerprints from OVF'.format( - len(pubkey_info) - if pubkey_info is not None - else 0 + pubkey_info = self.cfg.get("_pubkeys", None) + log_msg = "Retrieved {} fingerprints from OVF".format( + len(pubkey_info) if pubkey_info is not None else 0 ) report_diagnostic_event(log_msg, logger_func=LOG.debug) - metadata_func = partial(get_metadata_from_fabric, - fallback_lease_file=self. - dhclient_lease_file, - pubkey_info=pubkey_info) + metadata_func = partial( + get_metadata_from_fabric, + fallback_lease_file=self.dhclient_lease_file, + pubkey_info=pubkey_info, + ) LOG.debug("negotiating with fabric") try: @@ -1438,7 +1541,9 @@ class DataSourceAzure(sources.DataSource): except Exception as e: report_diagnostic_event( "Error communicating with Azure fabric; You may experience " - "connectivity issues: %s" % e, logger_func=LOG.warning) + "connectivity issues: %s" % e, + logger_func=LOG.warning, + ) return False util.del_file(REPORTED_READY_MARKER_FILE) @@ -1450,30 +1555,34 @@ class DataSourceAzure(sources.DataSource): @azure_ds_telemetry_reporter def activate(self, cfg, is_new_instance): try: - address_ephemeral_resize(is_new_instance=is_new_instance, - preserve_ntfs=self.ds_cfg.get( - DS_CFG_KEY_PRESERVE_NTFS, False)) + address_ephemeral_resize( + is_new_instance=is_new_instance, + preserve_ntfs=self.ds_cfg.get(DS_CFG_KEY_PRESERVE_NTFS, False), + ) finally: - push_log_to_kvp(self.sys_cfg['def_log_file']) + push_log_to_kvp(self.sys_cfg["def_log_file"]) return @property def availability_zone(self): - return self.metadata.get( - 'imds', {}).get('compute', {}).get('platformFaultDomain') + return ( + self.metadata.get("imds", {}) + .get("compute", {}) + .get("platformFaultDomain") + ) @property def network_config(self): """Generate a network config like net.generate_fallback_network() with - the following exceptions. + the following exceptions. - 1. Probe the drivers of the net-devices present and inject them in - the network configuration under params: driver: <driver> value - 2. Generate a fallback network config that does not include any of - the blacklisted devices. + 1. Probe the drivers of the net-devices present and inject them in + the network configuration under params: driver: <driver> value + 2. Generate a fallback network config that does not include any of + the blacklisted devices. """ if not self._network_config or self._network_config == sources.UNSET: - if self.ds_cfg.get('apply_network_config'): + if self.ds_cfg.get("apply_network_config"): nc_src = self._metadata_imds else: nc_src = None @@ -1482,33 +1591,36 @@ class DataSourceAzure(sources.DataSource): @property def region(self): - return self.metadata.get('imds', {}).get('compute', {}).get('location') + return self.metadata.get("imds", {}).get("compute", {}).get("location") def _username_from_imds(imds_data): try: - return imds_data['compute']['osProfile']['adminUsername'] + return imds_data["compute"]["osProfile"]["adminUsername"] except KeyError: return None def _userdata_from_imds(imds_data): try: - return imds_data['compute']['userData'] + return imds_data["compute"]["userData"] except KeyError: return None def _hostname_from_imds(imds_data): try: - return imds_data['compute']['osProfile']['computerName'] + return imds_data["compute"]["osProfile"]["computerName"] except KeyError: return None def _disable_password_from_imds(imds_data): try: - return imds_data['compute']['osProfile']['disablePasswordAuthentication'] == 'true' # noqa: E501 + return ( + imds_data["compute"]["osProfile"]["disablePasswordAuthentication"] + == "true" + ) except KeyError: return None @@ -1518,7 +1630,7 @@ def _key_is_openssh_formatted(key): Validate whether or not the key is OpenSSH-formatted. """ # See https://bugs.launchpad.net/cloud-init/+bug/1910835 - if '\r\n' in key.strip(): + if "\r\n" in key.strip(): return False parser = ssh_util.AuthKeyLineParser() @@ -1546,7 +1658,7 @@ def _partitions_on_device(devpath, maxnum=16): @azure_ds_telemetry_reporter def _has_ntfs_filesystem(devpath): ntfs_devices = util.find_devs_with("TYPE=ntfs", no_cache=True) - LOG.debug('ntfs_devices found = %s', ntfs_devices) + LOG.debug("ntfs_devices found = %s", ntfs_devices) return os.path.realpath(devpath) in ntfs_devices @@ -1570,24 +1682,29 @@ def can_dev_be_reformatted(devpath, preserve_ntfs): If cloud-init cannot mount the disk to check for data, destruction will be allowed, unless the dscfg key is set.""" if preserve_ntfs: - msg = ('config says to never destroy NTFS (%s.%s), skipping checks' % - (".".join(DS_CFG_PATH), DS_CFG_KEY_PRESERVE_NTFS)) + msg = "config says to never destroy NTFS (%s.%s), skipping checks" % ( + ".".join(DS_CFG_PATH), + DS_CFG_KEY_PRESERVE_NTFS, + ) return False, msg if not os.path.exists(devpath): - return False, 'device %s does not exist' % devpath + return False, "device %s does not exist" % devpath - LOG.debug('Resolving realpath of %s -> %s', devpath, - os.path.realpath(devpath)) + LOG.debug( + "Resolving realpath of %s -> %s", devpath, os.path.realpath(devpath) + ) # devpath of /dev/sd[a-z] or /dev/disk/cloud/azure_resource # where partitions are "<devpath>1" or "<devpath>-part1" or "<devpath>p1" partitions = _partitions_on_device(devpath) if len(partitions) == 0: - return False, 'device %s was not partitioned' % devpath + return False, "device %s was not partitioned" % devpath elif len(partitions) > 2: - msg = ('device %s had 3 or more partitions: %s' % - (devpath, ' '.join([p[1] for p in partitions]))) + msg = "device %s had 3 or more partitions: %s" % ( + devpath, + " ".join([p[1] for p in partitions]), + ) return False, msg elif len(partitions) == 2: cand_part, cand_path = partitions[1] @@ -1595,57 +1712,78 @@ def can_dev_be_reformatted(devpath, preserve_ntfs): cand_part, cand_path = partitions[0] if not _has_ntfs_filesystem(cand_path): - msg = ('partition %s (%s) on device %s was not ntfs formatted' % - (cand_part, cand_path, devpath)) + msg = "partition %s (%s) on device %s was not ntfs formatted" % ( + cand_part, + cand_path, + devpath, + ) return False, msg @azure_ds_telemetry_reporter def count_files(mp): - ignored = set(['dataloss_warning_readme.txt']) + ignored = set(["dataloss_warning_readme.txt"]) return len([f for f in os.listdir(mp) if f.lower() not in ignored]) - bmsg = ('partition %s (%s) on device %s was ntfs formatted' % - (cand_part, cand_path, devpath)) + bmsg = "partition %s (%s) on device %s was ntfs formatted" % ( + cand_part, + cand_path, + devpath, + ) with events.ReportEventStack( name="mount-ntfs-and-count", description="mount-ntfs-and-count", - parent=azure_ds_reporter + parent=azure_ds_reporter, ) as evt: try: - file_count = util.mount_cb(cand_path, count_files, mtype="ntfs", - update_env_for_mount={'LANG': 'C'}) + file_count = util.mount_cb( + cand_path, + count_files, + mtype="ntfs", + update_env_for_mount={"LANG": "C"}, + ) except util.MountFailedError as e: evt.description = "cannot mount ntfs" if "unknown filesystem type 'ntfs'" in str(e): - return True, (bmsg + ' but this system cannot mount NTFS,' - ' assuming there are no important files.' - ' Formatting allowed.') - return False, bmsg + ' but mount of %s failed: %s' % (cand_part, e) + return ( + True, + ( + bmsg + " but this system cannot mount NTFS," + " assuming there are no important files." + " Formatting allowed." + ), + ) + return False, bmsg + " but mount of %s failed: %s" % (cand_part, e) if file_count != 0: evt.description = "mounted and counted %d files" % file_count - LOG.warning("it looks like you're using NTFS on the ephemeral" - " disk, to ensure that filesystem does not get wiped," - " set %s.%s in config", '.'.join(DS_CFG_PATH), - DS_CFG_KEY_PRESERVE_NTFS) - return False, bmsg + ' but had %d files on it.' % file_count + LOG.warning( + "it looks like you're using NTFS on the ephemeral" + " disk, to ensure that filesystem does not get wiped," + " set %s.%s in config", + ".".join(DS_CFG_PATH), + DS_CFG_KEY_PRESERVE_NTFS, + ) + return False, bmsg + " but had %d files on it." % file_count - return True, bmsg + ' and had no important files. Safe for reformatting.' + return True, bmsg + " and had no important files. Safe for reformatting." @azure_ds_telemetry_reporter -def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, - is_new_instance=False, preserve_ntfs=False): +def address_ephemeral_resize( + devpath=RESOURCE_DISK_PATH, is_new_instance=False, preserve_ntfs=False +): if not os.path.exists(devpath): report_diagnostic_event( "Ephemeral resource disk '%s' does not exist." % devpath, - logger_func=LOG.debug) + logger_func=LOG.debug, + ) return else: report_diagnostic_event( "Ephemeral resource disk '%s' exists." % devpath, - logger_func=LOG.debug) + logger_func=LOG.debug, + ) result = False msg = None @@ -1658,31 +1796,32 @@ def address_ephemeral_resize(devpath=RESOURCE_DISK_PATH, if not result: return - for mod in ['disk_setup', 'mounts']: - sempath = '/var/lib/cloud/instance/sem/config_' + mod + for mod in ["disk_setup", "mounts"]: + sempath = "/var/lib/cloud/instance/sem/config_" + mod bmsg = 'Marker "%s" for module "%s"' % (sempath, mod) if os.path.exists(sempath): try: os.unlink(sempath) - LOG.debug('%s removed.', bmsg) + LOG.debug("%s removed.", bmsg) except Exception as e: # python3 throws FileNotFoundError, python2 throws OSError - LOG.warning('%s: remove failed! (%s)', bmsg, e) + LOG.warning("%s: remove failed! (%s)", bmsg, e) else: - LOG.debug('%s did not exist.', bmsg) + LOG.debug("%s did not exist.", bmsg) return @azure_ds_telemetry_reporter def write_files(datadir, files, dirmode=None): - def _redact_password(cnt, fname): """Azure provides the UserPassword in plain text. So we redact it""" try: root = ET.fromstring(cnt) for elem in root.iter(): - if ('UserPassword' in elem.tag and - elem.text != DEF_PASSWD_REDACTION): + if ( + "UserPassword" in elem.tag + and elem.text != DEF_PASSWD_REDACTION + ): elem.text = DEF_PASSWD_REDACTION return ET.tostring(root) except Exception: @@ -1696,7 +1835,7 @@ def write_files(datadir, files, dirmode=None): util.ensure_dir(datadir, dirmode) for (name, content) in files.items(): fname = os.path.join(datadir, name) - if 'ovf-env.xml' in name: + if "ovf-env.xml" in name: content = _redact_password(content, fname) util.write_file(filename=fname, content=content, mode=0o600) @@ -1728,8 +1867,9 @@ def load_azure_ovf_pubkeys(sshnode): if len(results) == 0: return [] if len(results) > 1: - raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" % - len(results)) + raise BrokenAzureDataSource( + "Multiple 'PublicKeys'(%s) in SSH node" % len(results) + ) pubkeys_node = results[0] pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey") @@ -1744,7 +1884,7 @@ def load_azure_ovf_pubkeys(sshnode): if not pk_node.hasChildNodes(): continue - cur = {'fingerprint': "", 'path': "", 'value': ""} + cur = {"fingerprint": "", "path": "", "value": ""} for child in pk_node.childNodes: if child.nodeType == text_node or not child.localName: continue @@ -1754,8 +1894,10 @@ def load_azure_ovf_pubkeys(sshnode): if name not in cur.keys(): continue - if (len(child.childNodes) != 1 or - child.childNodes[0].nodeType != text_node): + if ( + len(child.childNodes) != 1 + or child.childNodes[0].nodeType != text_node + ): continue cur[name] = child.childNodes[0].wholeText.strip() @@ -1773,33 +1915,37 @@ def read_azure_ovf(contents): report_diagnostic_event(error_str, logger_func=LOG.warning) raise BrokenAzureDataSource(error_str) from e - results = find_child(dom.documentElement, - lambda n: n.localName == "ProvisioningSection") + results = find_child( + dom.documentElement, lambda n: n.localName == "ProvisioningSection" + ) if len(results) == 0: raise NonAzureDataSource("No ProvisioningSection") if len(results) > 1: - raise BrokenAzureDataSource("found '%d' ProvisioningSection items" % - len(results)) + raise BrokenAzureDataSource( + "found '%d' ProvisioningSection items" % len(results) + ) provSection = results[0] - lpcs_nodes = find_child(provSection, - lambda n: - n.localName == "LinuxProvisioningConfigurationSet") + lpcs_nodes = find_child( + provSection, + lambda n: n.localName == "LinuxProvisioningConfigurationSet", + ) if len(lpcs_nodes) == 0: raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") if len(lpcs_nodes) > 1: - raise BrokenAzureDataSource("found '%d' %ss" % - (len(lpcs_nodes), - "LinuxProvisioningConfigurationSet")) + raise BrokenAzureDataSource( + "found '%d' %ss" + % (len(lpcs_nodes), "LinuxProvisioningConfigurationSet") + ) lpcs = lpcs_nodes[0] if not lpcs.hasChildNodes(): raise BrokenAzureDataSource("no child nodes of configuration set") - md_props = 'seedfrom' - md = {'azure_data': {}} + md_props = "seedfrom" + md = {"azure_data": {}} cfg = {} ud = "" password = None @@ -1813,8 +1959,10 @@ def read_azure_ovf(contents): simple = False value = "" - if (len(child.childNodes) == 1 and - child.childNodes[0].nodeType == dom.TEXT_NODE): + if ( + len(child.childNodes) == 1 + and child.childNodes[0].nodeType == dom.TEXT_NODE + ): simple = True value = child.childNodes[0].wholeText @@ -1823,8 +1971,8 @@ def read_azure_ovf(contents): # we accept either UserData or CustomData. If both are present # then behavior is undefined. if name == "userdata" or name == "customdata": - if attrs.get('encoding') in (None, "base64"): - ud = base64.b64decode(''.join(value.split())) + if attrs.get("encoding") in (None, "base64"): + ud = base64.b64decode("".join(value.split())) else: ud = value elif name == "username": @@ -1832,36 +1980,36 @@ def read_azure_ovf(contents): elif name == "userpassword": password = value elif name == "hostname": - md['local-hostname'] = value + md["local-hostname"] = value elif name == "dscfg": - if attrs.get('encoding') in (None, "base64"): - dscfg = base64.b64decode(''.join(value.split())) + if attrs.get("encoding") in (None, "base64"): + dscfg = base64.b64decode("".join(value.split())) else: dscfg = value - cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})} + cfg["datasource"] = {DS_NAME: util.load_yaml(dscfg, default={})} elif name == "ssh": - cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) + cfg["_pubkeys"] = load_azure_ovf_pubkeys(child) elif name == "disablesshpasswordauthentication": - cfg['ssh_pwauth'] = util.is_false(value) + cfg["ssh_pwauth"] = util.is_false(value) elif simple: if name in md_props: md[name] = value else: - md['azure_data'][name] = value + md["azure_data"][name] = value defuser = {} if username: - defuser['name'] = username + defuser["name"] = username if password: - defuser['lock_passwd'] = False + defuser["lock_passwd"] = False if DEF_PASSWD_REDACTION != password: - defuser['passwd'] = cfg['password'] = encrypt_pass(password) + defuser["passwd"] = cfg["password"] = encrypt_pass(password) if defuser: - cfg['system_info'] = {'default_user': defuser} + cfg["system_info"] = {"default_user": defuser} - if 'ssh_pwauth' not in cfg and password: - cfg['ssh_pwauth'] = True + if "ssh_pwauth" not in cfg and password: + cfg["ssh_pwauth"] = True preprovisioning_cfg = _get_preprovisioning_cfgs(dom) cfg = util.mergemanydict([cfg, preprovisioning_cfg]) @@ -1887,20 +2035,18 @@ def _get_preprovisioning_cfgs(dom): More specifically, this will never happen: - PreprovisionedVm=True and PreprovisionedVMType=Savable """ - cfg = { - "PreprovisionedVm": False, - "PreprovisionedVMType": None - } + cfg = {"PreprovisionedVm": False, "PreprovisionedVMType": None} platform_settings_section = find_child( - dom.documentElement, - lambda n: n.localName == "PlatformSettingsSection") + dom.documentElement, lambda n: n.localName == "PlatformSettingsSection" + ) if not platform_settings_section or len(platform_settings_section) == 0: LOG.debug("PlatformSettingsSection not found") return cfg platform_settings = find_child( platform_settings_section[0], - lambda n: n.localName == "PlatformSettings") + lambda n: n.localName == "PlatformSettings", + ) if not platform_settings or len(platform_settings) == 0: LOG.debug("PlatformSettings not found") return cfg @@ -1909,10 +2055,12 @@ def _get_preprovisioning_cfgs(dom): # platform has removed PreprovisionedVm and only surfaces # PreprovisionedVMType. cfg["PreprovisionedVm"] = _get_preprovisionedvm_cfg_value( - platform_settings) + platform_settings + ) cfg["PreprovisionedVMType"] = _get_preprovisionedvmtype_cfg_value( - platform_settings) + platform_settings + ) return cfg @@ -1924,16 +2072,18 @@ def _get_preprovisionedvm_cfg_value(platform_settings): # platform has removed PreprovisionedVm and only surfaces # PreprovisionedVMType. preprovisionedVmVal = find_child( - platform_settings[0], - lambda n: n.localName == "PreprovisionedVm") + platform_settings[0], lambda n: n.localName == "PreprovisionedVm" + ) if not preprovisionedVmVal or len(preprovisionedVmVal) == 0: LOG.debug("PreprovisionedVm not found") return preprovisionedVm preprovisionedVm = util.translate_bool( - preprovisionedVmVal[0].firstChild.nodeValue) + preprovisionedVmVal[0].firstChild.nodeValue + ) report_diagnostic_event( - "PreprovisionedVm: %s" % preprovisionedVm, logger_func=LOG.info) + "PreprovisionedVm: %s" % preprovisionedVm, logger_func=LOG.info + ) return preprovisionedVm @@ -1952,18 +2102,21 @@ def _get_preprovisionedvmtype_cfg_value(platform_settings): # Once assigned to customer, the customer-requested nics are # hot-attached to it and reprovision happens like today. preprovisionedVMTypeVal = find_child( - platform_settings[0], - lambda n: n.localName == "PreprovisionedVMType") - if (not preprovisionedVMTypeVal or len(preprovisionedVMTypeVal) == 0 or - preprovisionedVMTypeVal[0].firstChild is None): + platform_settings[0], lambda n: n.localName == "PreprovisionedVMType" + ) + if ( + not preprovisionedVMTypeVal + or len(preprovisionedVMTypeVal) == 0 + or preprovisionedVMTypeVal[0].firstChild is None + ): LOG.debug("PreprovisionedVMType not found") return preprovisionedVMType preprovisionedVMType = preprovisionedVMTypeVal[0].firstChild.nodeValue report_diagnostic_event( - "PreprovisionedVMType: %s" % preprovisionedVMType, - logger_func=LOG.info) + "PreprovisionedVMType: %s" % preprovisionedVMType, logger_func=LOG.info + ) return preprovisionedVMType @@ -1987,7 +2140,7 @@ def _check_freebsd_cdrom(cdrom_dev): @azure_ds_telemetry_reporter def _get_random_seed(source=PLATFORM_ENTROPY_SOURCE): """Return content random seed file if available, otherwise, - return None.""" + return None.""" # azure / hyper-v provides random data here # now update ds_cfg to reflect contents pass in config if source is None: @@ -2034,7 +2187,7 @@ def load_azure_ds_dir(source_dir): contents = fp.read() md, ud, cfg = read_azure_ovf(contents) - return (md, ud, cfg, {'ovf-env.xml': contents}) + return (md, ud, cfg, {"ovf-env.xml": contents}) @azure_ds_telemetry_reporter @@ -2051,12 +2204,14 @@ def parse_network_config(imds_metadata) -> dict: return _generate_network_config_from_imds_metadata(imds_metadata) except Exception as e: LOG.error( - 'Failed generating network config ' - 'from IMDS network metadata: %s', str(e)) + "Failed generating network config " + "from IMDS network metadata: %s", + str(e), + ) try: return _generate_network_config_from_fallback_config() except Exception as e: - LOG.error('Failed generating fallback network config: %s', str(e)) + LOG.error("Failed generating fallback network config: %s", str(e)) return {} @@ -2068,57 +2223,60 @@ def _generate_network_config_from_imds_metadata(imds_metadata) -> dict: @param: imds_metadata: Dict of content read from IMDS network service. @return: Dictionary containing network version 2 standard configuration. """ - netconfig = {'version': 2, 'ethernets': {}} - network_metadata = imds_metadata['network'] - for idx, intf in enumerate(network_metadata['interface']): + netconfig = {"version": 2, "ethernets": {}} + network_metadata = imds_metadata["network"] + for idx, intf in enumerate(network_metadata["interface"]): has_ip_address = False # First IPv4 and/or IPv6 address will be obtained via DHCP. # Any additional IPs of each type will be set as static # addresses. - nicname = 'eth{idx}'.format(idx=idx) - dhcp_override = {'route-metric': (idx + 1) * 100} - dev_config = {'dhcp4': True, 'dhcp4-overrides': dhcp_override, - 'dhcp6': False} - for addr_type in ('ipv4', 'ipv6'): - addresses = intf.get(addr_type, {}).get('ipAddress', []) + nicname = "eth{idx}".format(idx=idx) + dhcp_override = {"route-metric": (idx + 1) * 100} + dev_config = { + "dhcp4": True, + "dhcp4-overrides": dhcp_override, + "dhcp6": False, + } + for addr_type in ("ipv4", "ipv6"): + addresses = intf.get(addr_type, {}).get("ipAddress", []) # If there are no available IP addresses, then we don't # want to add this interface to the generated config. if not addresses: continue has_ip_address = True - if addr_type == 'ipv4': - default_prefix = '24' + if addr_type == "ipv4": + default_prefix = "24" else: - default_prefix = '128' + default_prefix = "128" if addresses: - dev_config['dhcp6'] = True + dev_config["dhcp6"] = True # non-primary interfaces should have a higher # route-metric (cost) so default routes prefer # primary nic due to lower route-metric value - dev_config['dhcp6-overrides'] = dhcp_override + dev_config["dhcp6-overrides"] = dhcp_override for addr in addresses[1:]: # Append static address config for ip > 1 - netPrefix = intf[addr_type]['subnet'][0].get( - 'prefix', default_prefix) - privateIp = addr['privateIpAddress'] - if not dev_config.get('addresses'): - dev_config['addresses'] = [] - dev_config['addresses'].append( - '{ip}/{prefix}'.format( - ip=privateIp, prefix=netPrefix)) + netPrefix = intf[addr_type]["subnet"][0].get( + "prefix", default_prefix + ) + privateIp = addr["privateIpAddress"] + if not dev_config.get("addresses"): + dev_config["addresses"] = [] + dev_config["addresses"].append( + "{ip}/{prefix}".format(ip=privateIp, prefix=netPrefix) + ) if dev_config and has_ip_address: - mac = ':'.join(re.findall(r'..', intf['macAddress'])) - dev_config.update({ - 'match': {'macaddress': mac.lower()}, - 'set-name': nicname - }) + mac = ":".join(re.findall(r"..", intf["macAddress"])) + dev_config.update( + {"match": {"macaddress": mac.lower()}, "set-name": nicname} + ) # With netvsc, we can get two interfaces that # share the same MAC, so we need to make sure # our match condition also contains the driver driver = device_driver(nicname) - if driver and driver == 'hv_netvsc': - dev_config['match']['driver'] = driver - netconfig['ethernets'][nicname] = dev_config + if driver and driver == "hv_netvsc": + dev_config["match"]["driver"] = driver + netconfig["ethernets"][nicname] = dev_config return netconfig @@ -2129,16 +2287,19 @@ def _generate_network_config_from_fallback_config() -> dict: @return: Dictionary containing network version 2 standard configuration. """ return net.generate_fallback_config( - blacklist_drivers=BLACKLIST_DRIVERS, config_driver=True) + blacklist_drivers=BLACKLIST_DRIVERS, config_driver=True + ) @azure_ds_telemetry_reporter -def get_metadata_from_imds(fallback_nic, - retries, - md_type=metadata_type.all, - api_version=IMDS_VER_MIN, - exc_cb=retry_on_url_exc, - infinite=False): +def get_metadata_from_imds( + fallback_nic, + retries, + md_type=metadata_type.all, + api_version=IMDS_VER_MIN, + exc_cb=retry_on_url_exc, + infinite=False, +): """Query Azure's instance metadata service, returning a dictionary. If network is not up, setup ephemeral dhcp on fallback_nic to talk to the @@ -2154,31 +2315,34 @@ def get_metadata_from_imds(fallback_nic, @return: A dict of instance metadata containing compute and network info. """ - kwargs = {'logfunc': LOG.debug, - 'msg': 'Crawl of Azure Instance Metadata Service (IMDS)', - 'func': _get_metadata_from_imds, - 'args': (retries, exc_cb, md_type, api_version, infinite)} + kwargs = { + "logfunc": LOG.debug, + "msg": "Crawl of Azure Instance Metadata Service (IMDS)", + "func": _get_metadata_from_imds, + "args": (retries, exc_cb, md_type, api_version, infinite), + } if net.is_up(fallback_nic): return util.log_time(**kwargs) else: try: - with EphemeralDHCPv4WithReporting( - azure_ds_reporter, fallback_nic): + with EphemeralDHCPv4WithReporting(azure_ds_reporter, fallback_nic): return util.log_time(**kwargs) except Exception as e: report_diagnostic_event( "exception while getting metadata: %s" % e, - logger_func=LOG.warning) + logger_func=LOG.warning, + ) raise @azure_ds_telemetry_reporter def _get_metadata_from_imds( - retries, - exc_cb, - md_type=metadata_type.all, - api_version=IMDS_VER_MIN, - infinite=False): + retries, + exc_cb, + md_type=metadata_type.all, + api_version=IMDS_VER_MIN, + infinite=False, +): url = "{}?api-version={}".format(md_type.value, api_version) headers = {"Metadata": "true"} @@ -2188,20 +2352,27 @@ def _get_metadata_from_imds( try: response = readurl( - url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers, - retries=retries, exception_cb=exc_cb, infinite=infinite) + url, + timeout=IMDS_TIMEOUT_IN_SECONDS, + headers=headers, + retries=retries, + exception_cb=exc_cb, + infinite=infinite, + ) except Exception as e: # pylint:disable=no-member if isinstance(e, UrlError) and e.code == 400: raise else: report_diagnostic_event( - 'Ignoring IMDS instance metadata. ' - 'Get metadata from IMDS failed: %s' % e, - logger_func=LOG.warning) + "Ignoring IMDS instance metadata. " + "Get metadata from IMDS failed: %s" % e, + logger_func=LOG.warning, + ) return {} try: from json.decoder import JSONDecodeError + json_decode_error = JSONDecodeError except ImportError: json_decode_error = ValueError @@ -2210,9 +2381,10 @@ def _get_metadata_from_imds( return util.load_json(str(response)) except json_decode_error as e: report_diagnostic_event( - 'Ignoring non-json IMDS instance metadata response: %s. ' - 'Loading non-json IMDS response failed: %s' % (str(response), e), - logger_func=LOG.warning) + "Ignoring non-json IMDS instance metadata response: %s. " + "Loading non-json IMDS response failed: %s" % (str(response), e), + logger_func=LOG.warning, + ) return {} @@ -2242,10 +2414,11 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): if os.path.exists(path): if not logged: LOG.info( - 'Removing Ubuntu extended network scripts because' - ' cloud-init updates Azure network configuration on the' - ' following events: %s.', - [EventType.BOOT.value, EventType.BOOT_LEGACY.value]) + "Removing Ubuntu extended network scripts because" + " cloud-init updates Azure network configuration on the" + " following events: %s.", + [EventType.BOOT.value, EventType.BOOT_LEGACY.value], + ) logged = True if os.path.isdir(path): util.del_dir(path) @@ -2258,15 +2431,15 @@ def _is_platform_viable(seed_dir): with events.ReportEventStack( name="check-platform-viability", description="found azure asset tag", - parent=azure_ds_reporter + parent=azure_ds_reporter, ) as evt: - asset_tag = dmi.read_dmi_data('chassis-asset-tag') + asset_tag = dmi.read_dmi_data("chassis-asset-tag") if asset_tag == AZURE_CHASSIS_ASSET_TAG: return True msg = "Non-Azure DMI asset tag '%s' discovered." % asset_tag evt.description = msg report_diagnostic_event(msg, logger_func=LOG.debug) - if os.path.exists(os.path.join(seed_dir, 'ovf-env.xml')): + if os.path.exists(os.path.join(seed_dir, "ovf-env.xml")): return True return False @@ -2284,7 +2457,7 @@ DataSourceAzureNet = DataSourceAzure # Used to match classes to dependencies datasources = [ - (DataSourceAzure, (sources.DEP_FILESYSTEM, )), + (DataSourceAzure, (sources.DEP_FILESYSTEM,)), ] @@ -2292,4 +2465,5 @@ datasources = [ def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + # vi: ts=4 expandtab |