diff options
Diffstat (limited to 'cloudinit/sources/DataSourceSmartOS.py')
| -rw-r--r-- | cloudinit/sources/DataSourceSmartOS.py | 781 |
1 files changed, 0 insertions, 781 deletions
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py deleted file mode 100644 index ccc86883..00000000 --- a/cloudinit/sources/DataSourceSmartOS.py +++ /dev/null @@ -1,781 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2013 Canonical Ltd. -# -# Author: Ben Howard <ben.howard@canonical.com> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. -# -# -# Datasource for provisioning on SmartOS. This works on Joyent -# and public/private Clouds using SmartOS. -# -# SmartOS hosts use a serial console (/dev/ttyS1) on KVM Linux Guests -# The meta-data is transmitted via key/value pairs made by -# requests on the console. For example, to get the hostname, you -# would send "GET hostname" on /dev/ttyS1. -# For Linux Guests running in LX-Brand Zones on SmartOS hosts -# a socket (/native/.zonecontrol/metadata.sock) is used instead -# of a serial console. -# -# Certain behavior is defined by the DataDictionary -# http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html -# Comments with "@datadictionary" are snippets of the definition - -import base64 -import binascii -import json -import os -import random -import re -import socket - -from cloudinit import log as logging -from cloudinit import serial -from cloudinit import sources -from cloudinit import util - -LOG = logging.getLogger(__name__) - -SMARTOS_ATTRIB_MAP = { - # Cloud-init Key : (SmartOS Key, Strip line endings) - 'instance-id': ('sdc:uuid', True), - 'local-hostname': ('hostname', True), - 'public-keys': ('root_authorized_keys', True), - 'user-script': ('user-script', False), - 'legacy-user-data': ('user-data', False), - 'user-data': ('cloud-init:user-data', False), - 'iptables_disable': ('iptables_disable', True), - 'motd_sys_info': ('motd_sys_info', True), - 'availability_zone': ('sdc:datacenter_name', True), - 'vendor-data': ('sdc:vendor-data', False), - 'operator-script': ('sdc:operator-script', False), -} - -SMARTOS_ATTRIB_JSON = { - # Cloud-init Key : (SmartOS Key known JSON) - 'network-data': 'sdc:nics', -} - -SMARTOS_ENV_LX_BRAND = "lx-brand" -SMARTOS_ENV_KVM = "kvm" - -DS_NAME = 'SmartOS' -DS_CFG_PATH = ['datasource', DS_NAME] -NO_BASE64_DECODE = [ - 'iptables_disable', - 'motd_sys_info', - 'root_authorized_keys', - 'sdc:datacenter_name', - 'sdc:uuid' - 'user-data', - 'user-script', -] - -METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock' -SERIAL_DEVICE = '/dev/ttyS1' -SERIAL_TIMEOUT = 60 - -# BUILT-IN DATASOURCE CONFIGURATION -# The following is the built-in configuration. If the values -# are not set via the system configuration, then these default -# will be used: -# serial_device: which serial device to use for the meta-data -# serial_timeout: how long to wait on the device -# no_base64_decode: values which are not base64 encoded and -# are fetched directly from SmartOS, not meta-data values -# base64_keys: meta-data keys that are delivered in base64 -# base64_all: with the exclusion of no_base64_decode values, -# treat all meta-data as base64 encoded -# disk_setup: describes how to partition the ephemeral drive -# fs_setup: describes how to format the ephemeral drive -# -BUILTIN_DS_CONFIG = { - 'serial_device': SERIAL_DEVICE, - 'serial_timeout': SERIAL_TIMEOUT, - 'metadata_sockfile': METADATA_SOCKFILE, - 'no_base64_decode': NO_BASE64_DECODE, - 'base64_keys': [], - 'base64_all': False, - 'disk_aliases': {'ephemeral0': '/dev/vdb'}, -} - -BUILTIN_CLOUD_CONFIG = { - 'disk_setup': { - 'ephemeral0': {'table_type': 'mbr', - 'layout': False, - 'overwrite': False} - }, - 'fs_setup': [{'label': 'ephemeral0', - 'filesystem': 'ext3', - 'device': 'ephemeral0'}], -} - -# builtin vendor-data is a boothook that writes a script into -# /var/lib/cloud/scripts/per-boot. *That* script then handles -# executing the 'operator-script' and 'user-script' files -# that cloud-init writes into /var/lib/cloud/instance/data/ -# if they exist. -# -# This is all very indirect, but its done like this so that at -# some point in the future, perhaps cloud-init wouldn't do it at -# all, but rather the vendor actually provide vendor-data that accomplished -# their desires. (That is the point of vendor-data). -# -# cloud-init does cheat a bit, and write the operator-script and user-script -# itself. It could have the vendor-script do that, but it seems better -# to not require the image to contain a tool (mdata-get) to read those -# keys when we have a perfectly good one inside cloud-init. -BUILTIN_VENDOR_DATA = """\ -#cloud-boothook -#!/bin/sh -fname="%(per_boot_d)s/01_smartos_vendor_data.sh" -mkdir -p "${fname%%/*}" -cat > "$fname" <<"END_SCRIPT" -#!/bin/sh -## -# This file is written as part of the default vendor data for SmartOS. -# The SmartOS datasource writes the listed file from the listed metadata key -# sdc:operator-script -> %(operator_script)s -# user-script -> %(user_script)s -# -# You can view content with 'mdata-get <key>' -# -for script in "%(operator_script)s" "%(user_script)s"; do - [ -x "$script" ] || continue - echo "executing '$script'" 1>&2 - "$script" -done -END_SCRIPT -chmod +x "$fname" -""" - - -# @datadictionary: this is legacy path for placing files from metadata -# per the SmartOS location. It is not preferable, but is done for -# legacy reasons -LEGACY_USER_D = "/var/db" - - -class DataSourceSmartOS(sources.DataSource): - _unset = "_unset" - smartos_type = _unset - md_client = _unset - - def __init__(self, sys_cfg, distro, paths): - sources.DataSource.__init__(self, sys_cfg, distro, paths) - self.ds_cfg = util.mergemanydict([ - self.ds_cfg, - util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), - BUILTIN_DS_CONFIG]) - - self.metadata = {} - self.network_data = None - self._network_config = None - - self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) - - self._init() - - def __str__(self): - root = sources.DataSource.__str__(self) - return "%s [client=%s]" % (root, self.md_client) - - def _init(self): - if self.smartos_type == self._unset: - self.smartos_type = get_smartos_environ() - if self.smartos_type is None: - self.md_client = None - - if self.md_client == self._unset: - self.md_client = jmc_client_factory( - smartos_type=self.smartos_type, - metadata_sockfile=self.ds_cfg['metadata_sockfile'], - serial_device=self.ds_cfg['serial_device'], - serial_timeout=self.ds_cfg['serial_timeout']) - - def _set_provisioned(self): - '''Mark the instance provisioning state as successful. - - When run in a zone, the host OS will look for /var/svc/provisioning - to be renamed as /var/svc/provision_success. This should be done - after meta-data is successfully retrieved and from this point - the host considers the provision of the zone to be a success and - keeps the zone running. - ''' - - LOG.debug('Instance provisioning state set as successful') - svc_path = '/var/svc' - if os.path.exists('/'.join([svc_path, 'provisioning'])): - os.rename('/'.join([svc_path, 'provisioning']), - '/'.join([svc_path, 'provision_success'])) - - def get_data(self): - self._init() - - md = {} - ud = "" - - if not self.smartos_type: - LOG.debug("Not running on smartos") - return False - - if not self.md_client.exists(): - LOG.debug("No metadata device '%r' found for SmartOS datasource", - self.md_client) - return False - - for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items(): - smartos_noun, strip = attribute - md[ci_noun] = self.md_client.get(smartos_noun, strip=strip) - - for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items(): - md[ci_noun] = self.md_client.get_json(smartos_noun) - - # @datadictionary: This key may contain a program that is written - # to a file in the filesystem of the guest on each boot and then - # executed. It may be of any format that would be considered - # executable in the guest instance. - # - # We write 'user-script' and 'operator-script' into the - # instance/data directory. The default vendor-data then handles - # executing them later. - data_d = os.path.join(self.paths.get_cpath(), 'instances', - md['instance-id'], 'data') - user_script = os.path.join(data_d, 'user-script') - u_script_l = "%s/user-script" % LEGACY_USER_D - write_boot_content(md.get('user-script'), content_f=user_script, - link=u_script_l, shebang=True, mode=0o700) - - operator_script = os.path.join(data_d, 'operator-script') - write_boot_content(md.get('operator-script'), - content_f=operator_script, shebang=False, - mode=0o700) - - # @datadictionary: This key has no defined format, but its value - # is written to the file /var/db/mdata-user-data on each boot prior - # to the phase that runs user-script. This file is not to be executed. - # This allows a configuration file of some kind to be injected into - # the machine to be consumed by the user-script when it runs. - u_data = md.get('legacy-user-data') - u_data_f = "%s/mdata-user-data" % LEGACY_USER_D - write_boot_content(u_data, u_data_f) - - # Handle the cloud-init regular meta - if not md['local-hostname']: - md['local-hostname'] = md['instance-id'] - - ud = None - if md['user-data']: - ud = md['user-data'] - - if not md['vendor-data']: - md['vendor-data'] = BUILTIN_VENDOR_DATA % { - 'user_script': user_script, - 'operator_script': operator_script, - 'per_boot_d': os.path.join(self.paths.get_cpath("scripts"), - 'per-boot'), - } - - self.metadata = util.mergemanydict([md, self.metadata]) - self.userdata_raw = ud - self.vendordata_raw = md['vendor-data'] - self.network_data = md['network-data'] - - self._set_provisioned() - return True - - def device_name_to_device(self, name): - return self.ds_cfg['disk_aliases'].get(name) - - def get_config_obj(self): - if self.smartos_type == SMARTOS_ENV_KVM: - return BUILTIN_CLOUD_CONFIG - return {} - - def get_instance_id(self): - return self.metadata['instance-id'] - - @property - def network_config(self): - if self._network_config is None: - if self.network_data is not None: - self._network_config = ( - convert_smartos_network_data(self.network_data)) - return self._network_config - - -class JoyentMetadataFetchException(Exception): - pass - - -class JoyentMetadataClient(object): - """ - A client implementing v2 of the Joyent Metadata Protocol Specification. - - The full specification can be found at - http://eng.joyent.com/mdata/protocol.html - """ - line_regex = re.compile( - r'V2 (?P<length>\d+) (?P<checksum>[0-9a-f]+)' - r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)' - r'( (?P<payload>.+))?)') - - def __init__(self, smartos_type=None, fp=None): - if smartos_type is None: - smartos_type = get_smartos_environ() - self.smartos_type = smartos_type - self.fp = fp - - def _checksum(self, body): - return '{0:08x}'.format( - binascii.crc32(body.encode('utf-8')) & 0xffffffff) - - def _get_value_from_frame(self, expected_request_id, frame): - frame_data = self.line_regex.match(frame).groupdict() - if int(frame_data['length']) != len(frame_data['body']): - raise JoyentMetadataFetchException( - 'Incorrect frame length given ({0} != {1}).'.format( - frame_data['length'], len(frame_data['body']))) - expected_checksum = self._checksum(frame_data['body']) - if frame_data['checksum'] != expected_checksum: - raise JoyentMetadataFetchException( - 'Invalid checksum (expected: {0}; got {1}).'.format( - expected_checksum, frame_data['checksum'])) - if frame_data['request_id'] != expected_request_id: - raise JoyentMetadataFetchException( - 'Request ID mismatch (expected: {0}; got {1}).'.format( - expected_request_id, frame_data['request_id'])) - if not frame_data.get('payload', None): - LOG.debug('No value found.') - return None - value = util.b64d(frame_data['payload']) - LOG.debug('Value "%s" found.', value) - return value - - def request(self, rtype, param=None): - request_id = '{0:08x}'.format(random.randint(0, 0xffffffff)) - message_body = ' '.join((request_id, rtype,)) - if param: - message_body += ' ' + base64.b64encode(param.encode()).decode() - msg = 'V2 {0} {1} {2}\n'.format( - len(message_body), self._checksum(message_body), message_body) - LOG.debug('Writing "%s" to metadata transport.', msg) - - need_close = False - if not self.fp: - self.open_transport() - need_close = True - - self.fp.write(msg.encode('ascii')) - self.fp.flush() - - response = bytearray() - response.extend(self.fp.read(1)) - while response[-1:] != b'\n': - response.extend(self.fp.read(1)) - - if need_close: - self.close_transport() - - response = response.rstrip().decode('ascii') - LOG.debug('Read "%s" from metadata transport.', response) - - if 'SUCCESS' not in response: - return None - - value = self._get_value_from_frame(request_id, response) - return value - - def get(self, key, default=None, strip=False): - result = self.request(rtype='GET', param=key) - if result is None: - return default - if result and strip: - result = result.strip() - return result - - def get_json(self, key, default=None): - result = self.get(key, default=default) - if result is None: - return default - return json.loads(result) - - def list(self): - result = self.request(rtype='KEYS') - if result: - result = result.split('\n') - return result - - def put(self, key, val): - param = b' '.join([base64.b64encode(i.encode()) - for i in (key, val)]).decode() - return self.request(rtype='PUT', param=param) - - def delete(self, key): - return self.request(rtype='DELETE', param=key) - - def close_transport(self): - if self.fp: - self.fp.close() - self.fp = None - - def __enter__(self): - if self.fp: - return self - self.open_transport() - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close_transport() - return - - def open_transport(self): - raise NotImplementedError - - -class JoyentMetadataSocketClient(JoyentMetadataClient): - def __init__(self, socketpath): - self.socketpath = socketpath - - def open_transport(self): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.connect(self.socketpath) - self.fp = sock.makefile('rwb') - - def exists(self): - return os.path.exists(self.socketpath) - - def __repr__(self): - return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath) - - -class JoyentMetadataSerialClient(JoyentMetadataClient): - def __init__(self, device, timeout=10, smartos_type=None): - super(JoyentMetadataSerialClient, self).__init__(smartos_type) - self.device = device - self.timeout = timeout - - def exists(self): - return os.path.exists(self.device) - - def open_transport(self): - ser = serial.Serial(self.device, timeout=self.timeout) - if not ser.isOpen(): - raise SystemError("Unable to open %s" % self.device) - self.fp = ser - - def __repr__(self): - return "%s(device=%s, timeout=%s)" % ( - self.__class__.__name__, self.device, self.timeout) - - -class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient): - """V1 of the protocol was not safe for all values. - Thus, we allowed the user to pass values in as base64 encoded. - Users may still reasonably expect to be able to send base64 data - and have it transparently decoded. So even though the V2 format is - now used, and is safe (using base64 itself), we keep legacy support. - - The way for a user to do this was: - a.) specify 'base64_keys' key whose value is a comma delimited - list of keys that were base64 encoded. - b.) base64_all: string interpreted as a boolean that indicates - if all keys are base64 encoded. - c.) set a key named b64-<keyname> with a boolean indicating that - <keyname> is base64 encoded.""" - - def __init__(self, device, timeout=10, smartos_type=None): - s = super(JoyentMetadataLegacySerialClient, self) - s.__init__(device, timeout, smartos_type) - self.base64_keys = None - self.base64_all = None - - def _init_base64_keys(self, reset=False): - if reset: - self.base64_keys = None - self.base64_all = None - - keys = None - if self.base64_all is None: - keys = self.list() - if 'base64_all' in keys: - self.base64_all = util.is_true(self._get("base64_all")) - else: - self.base64_all = False - - if self.base64_all: - # short circuit if base64_all is true - return - - if self.base64_keys is None: - if keys is None: - keys = self.list() - b64_keys = set() - if 'base64_keys' in keys: - b64_keys = set(self._get("base64_keys").split(",")) - - # now add any b64-<keyname> that has a true value - for key in [k[3:] for k in keys if k.startswith("b64-")]: - if util.is_true(self._get(key)): - b64_keys.add(key) - else: - if key in b64_keys: - b64_keys.remove(key) - - self.base64_keys = b64_keys - - def _get(self, key, default=None, strip=False): - return (super(JoyentMetadataLegacySerialClient, self). - get(key, default=default, strip=strip)) - - def is_b64_encoded(self, key, reset=False): - if key in NO_BASE64_DECODE: - return False - - self._init_base64_keys(reset=reset) - if self.base64_all: - return True - - return key in self.base64_keys - - def get(self, key, default=None, strip=False): - mdefault = object() - val = self._get(key, strip=False, default=mdefault) - if val is mdefault: - return default - - if self.is_b64_encoded(key): - try: - val = base64.b64decode(val.encode()).decode() - # Bogus input produces different errors in Python 2 and 3 - except (TypeError, binascii.Error): - LOG.warn("Failed base64 decoding key '%s': %s", key, val) - - if strip: - val = val.strip() - - return val - - -def jmc_client_factory( - smartos_type=None, metadata_sockfile=METADATA_SOCKFILE, - serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT, - uname_version=None): - - if smartos_type is None: - smartos_type = get_smartos_environ(uname_version) - - if smartos_type is None: - return None - elif smartos_type == SMARTOS_ENV_KVM: - return JoyentMetadataLegacySerialClient( - device=serial_device, timeout=serial_timeout, - smartos_type=smartos_type) - elif smartos_type == SMARTOS_ENV_LX_BRAND: - return JoyentMetadataSocketClient(socketpath=metadata_sockfile) - - raise ValueError("Unknown value for smartos_type: %s" % smartos_type) - - -def write_boot_content(content, content_f, link=None, shebang=False, - mode=0o400): - """ - Write the content to content_f. Under the following rules: - 1. If no content, remove the file - 2. Write the content - 3. If executable and no file magic, add it - 4. If there is a link, create it - - @param content: what to write - @param content_f: the file name - @param backup_d: the directory to save the backup at - @param link: if defined, location to create a symlink to - @param shebang: if no file magic, set shebang - @param mode: file mode - - Becuase of the way that Cloud-init executes scripts (no shell), - a script will fail to execute if does not have a magic bit (shebang) set - for the file. If shebang=True, then the script will be checked for a magic - bit and to the SmartOS default of assuming that bash. - """ - - if not content and os.path.exists(content_f): - os.unlink(content_f) - if link and os.path.islink(link): - os.unlink(link) - if not content: - return - - util.write_file(content_f, content, mode=mode) - - if shebang and not content.startswith("#!"): - try: - cmd = ["file", "--brief", "--mime-type", content_f] - (f_type, _err) = util.subp(cmd) - LOG.debug("script %s mime type is %s", content_f, f_type) - if f_type.strip() == "text/plain": - new_content = "\n".join(["#!/bin/bash", content]) - util.write_file(content_f, new_content, mode=mode) - LOG.debug("added shebang to file %s", content_f) - - except Exception as e: - util.logexc(LOG, ("Failed to identify script type for %s" % - content_f, e)) - - if link: - try: - if os.path.islink(link): - os.unlink(link) - if content and os.path.exists(content_f): - util.ensure_dir(os.path.dirname(link)) - os.symlink(content_f, link) - except IOError as e: - util.logexc(LOG, "failed establishing content link: %s", e) - - -def get_smartos_environ(uname_version=None, product_name=None, - uname_arch=None): - uname = os.uname() - if uname_arch is None: - uname_arch = uname[4] - - if uname_arch.startswith("arm") or uname_arch == "aarch64": - return None - - # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but - # report 'BrandZ virtual linux' as the kernel version - if uname_version is None: - uname_version = uname[3] - if uname_version.lower() == 'brandz virtual linux': - return SMARTOS_ENV_LX_BRAND - - if product_name is None: - system_type = util.read_dmi_data("system-product-name") - else: - system_type = product_name - - if system_type and 'smartdc' in system_type.lower(): - return SMARTOS_ENV_KVM - - return None - - -# Covert SMARTOS 'sdc:nics' data to network_config yaml -def convert_smartos_network_data(network_data=None): - """Return a dictionary of network_config by parsing provided - SMARTOS sdc:nics configuration data - - sdc:nics data is a dictionary of properties of a nic and the ip - configuration desired. Additional nic dictionaries are appended - to the list. - - Converting the format is straightforward though it does include - duplicate information as well as data which appears to be relevant - to the hostOS rather than the guest. - - For each entry in the nics list returned from query sdc:nics, we - create a type: physical entry, and extract the interface properties: - 'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'. The remaining - keys are related to ip configuration. For each ip in the 'ips' list - we create a subnet entry under 'subnets' pairing the ip to a one in - the 'gateways' list. - """ - - valid_keys = { - 'physical': [ - 'mac_address', - 'mtu', - 'name', - 'params', - 'subnets', - 'type', - ], - 'subnet': [ - 'address', - 'broadcast', - 'dns_nameservers', - 'dns_search', - 'gateway', - 'metric', - 'netmask', - 'pointopoint', - 'routes', - 'scope', - 'type', - ], - } - - config = [] - for nic in network_data: - cfg = dict((k, v) for k, v in nic.items() - if k in valid_keys['physical']) - cfg.update({ - 'type': 'physical', - 'name': nic['interface']}) - if 'mac' in nic: - cfg.update({'mac_address': nic['mac']}) - - subnets = [] - for ip, gw in zip(nic['ips'], nic['gateways']): - subnet = dict((k, v) for k, v in nic.items() - if k in valid_keys['subnet']) - subnet.update({ - 'type': 'static', - 'address': ip, - 'gateway': gw, - }) - subnets.append(subnet) - cfg.update({'subnets': subnets}) - config.append(cfg) - - return {'version': 1, 'config': config} - - -# Used to match classes to dependencies -datasources = [ - (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )), -] - - -# Return a list of data sources that match this set of dependencies -def get_datasource_list(depends): - return sources.list_from_depends(depends, datasources) - - -if __name__ == "__main__": - import sys - jmc = jmc_client_factory() - if jmc is None: - print("Do not appear to be on smartos.") - sys.exit(1) - if len(sys.argv) == 1: - keys = (list(SMARTOS_ATTRIB_JSON.keys()) + - list(SMARTOS_ATTRIB_MAP.keys())) - else: - keys = sys.argv[1:] - - data = {} - for key in keys: - if key in SMARTOS_ATTRIB_JSON: - keyname = SMARTOS_ATTRIB_JSON[key] - data[key] = jmc.get_json(keyname) - else: - if key in SMARTOS_ATTRIB_MAP: - keyname, strip = SMARTOS_ATTRIB_MAP[key] - else: - keyname, strip = (key, False) - val = jmc.get(keyname, strip=strip) - data[key] = jmc.get(keyname, strip=strip) - - print(json.dumps(data, indent=1)) |