From 45ac9be7c3090fa137b9bf711742123e4f92d5a3 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 3 Jun 2015 13:24:40 +0000 Subject: Stop extensions/writeexts.py from depending on morphlib and cliapp Change-Id: I2a53e250f1782e23f7be20e7a1514c392180ec02 --- extensions/writeexts.py | 170 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 121 insertions(+), 49 deletions(-) diff --git a/extensions/writeexts.py b/extensions/writeexts.py index aa185a2b..61d40789 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -13,19 +13,84 @@ # with this program. If not, see . -import cliapp +import contextlib +import errno import logging import os import re import shutil +import stat +import subprocess import sys -import time import tempfile -import errno -import stat -import contextlib +import time + + +def shell_quote(string): + '''Return a shell-quoted version of `string`.''' + lower_ascii = 'abcdefghijklmnopqrstuvwxyz' + upper_ascii = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + digits = '0123456789' + punctuation = '-_/=.,:' + safe = set(lower_ascii + upper_ascii + digits + punctuation) + + quoted = [] + for character in string: + if character in safe: + quoted.append(character) + elif character == "'": + quoted.append('"\'"') + else: + quoted.append("'%c'" % character) + + return ''.join(quoted) + + +def run_ssh_command(host, command): + '''Run `command` over SSH on `host`.''' + ssh_cmd = ['ssh', host, '--'] + [shell_quote(arg) for arg in command] + return subprocess.check_output(ssh_cmd) + + +def write_from_dict(filepath, d, validate=lambda x, y: True): + '''Takes a dictionary and appends the contents to a file + + An optional validation callback can be passed to perform validation on + each value in the dictionary. -import morphlib + e.g. + + def validation_callback(dictionary_key, dictionary_value): + if not dictionary_value.isdigit(): + raise Exception('value contains non-digit character(s)') + + Any callback supplied to this function should raise an exception + if validation fails. + ''' + + # Sort items asciibetically + # the output of the deployment should not depend + # on the locale of the machine running the deployment + items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) + + for (k, v) in items: + validate(k, v) + + with open(filepath, 'a') as f: + for (_, v) in items: + f.write('%s\n' % v) + + os.fchown(f.fileno(), 0, 0) + os.fchmod(f.fileno(), 0644) + + +class ExtensionError(Exception): + + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg class Fstab(object): @@ -75,11 +140,13 @@ class Fstab(object): def write(self): '''Rewrite the fstab file to include all new entries.''' - with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + with tempfile.NamedTemporaryFile(delete=False) as f: f.write(self.text) + tmp = f.name + shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) -class WriteExtension(cliapp.Application): +class WriteExtension(object): '''A base class for deployment write extensions. @@ -98,8 +165,6 @@ class WriteExtension(cliapp.Application): This file descriptor is read by Morph and written into its own log file. - This overrides cliapp's usual configurable logging setup. - ''' log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) @@ -115,13 +180,19 @@ class WriteExtension(cliapp.Application): logger.addHandler(handler) logger.setLevel(logging.DEBUG) - def log_config(self): - with morphlib.util.hide_password_environment_variables(os.environ): - cliapp.Application.log_config(self) - def process_args(self, args): raise NotImplementedError() + def run(self, args=None): + if args is None: + args = sys.argv[1:] + try: + self.setup_logging() + self.process_args(args) + except ExtensionError as e: + sys.stdout.write('ERROR: %s' % e) + sys.exit(1) + def status(self, **kwargs): '''Provide status output. @@ -130,9 +201,8 @@ class WriteExtension(cliapp.Application): by %. ''' - - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - self.output.flush() + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.flush() def check_for_btrfs_in_deployment_host_kernel(self): with open('/proc/filesystems') as f: @@ -141,7 +211,7 @@ class WriteExtension(cliapp.Application): def require_btrfs_in_deployment_host_kernel(self): if not self.check_for_btrfs_in_deployment_host_kernel(): - raise cliapp.AppException( + raise ExtensionError( 'Error: Btrfs is required for this deployment, but was not ' 'detected in the kernel of the machine that is running Morph.') @@ -156,7 +226,7 @@ class WriteExtension(cliapp.Application): def created_disk_image(self, location): size = self.get_disk_size() if not size: - raise cliapp.AppException('DISK_SIZE is not defined') + raise ExtensionError('DISK_SIZE is not defined') self.create_raw_disk_image(location, size) try: yield @@ -210,7 +280,7 @@ class WriteExtension(cliapp.Application): return None bytes = self._parse_size(size) if bytes is None: - raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + raise ExtensionError('Cannot parse %s value %s' % (env_var, size)) return bytes def get_disk_size(self): @@ -243,15 +313,15 @@ class WriteExtension(cliapp.Application): # need to do this because at the time of writing, SYSLINUX has not # been updated to understand these new features and will fail to # boot if the kernel is on a filesystem where they are enabled. - cliapp.runcmd( + subprocess.check_output( ['mkfs.btrfs','-f', '-L', 'baserock', '--features', '^extref', '--features', '^skinny-metadata', '--features', '^mixed-bg', '--nodesize', '4096', location]) - except cliapp.AppException as e: - if 'unrecognized option \'--features\'' in e.msg: + except subprocess.CalledProcessError as e: + if 'unrecognized option \'--features\'' in e.output: # Old versions of mkfs.btrfs (including v0.20, present in many # Baserock releases) don't support the --features option, but # also don't enable the new features by default. So we can @@ -259,7 +329,8 @@ class WriteExtension(cliapp.Application): logging.debug( 'Assuming mkfs.btrfs failure was because the tool is too ' 'old to have --features flag.') - cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + subprocess.check_call( + ['mkfs.btrfs','-f', '-L', 'baserock', location]) else: raise @@ -267,8 +338,8 @@ class WriteExtension(cliapp.Application): '''Get the UUID of a block device's file system.''' # Requires util-linux blkid; busybox one ignores options and # lies by exiting successfully. - return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', - location]).strip() + return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() @contextlib.contextmanager def mount(self, location): @@ -276,9 +347,10 @@ class WriteExtension(cliapp.Application): try: mount_point = tempfile.mkdtemp() if self.is_device(location): - cliapp.runcmd(['mount', location, mount_point]) + subprocess.check_call(['mount', location, mount_point]) else: - cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + subprocess.check_call(['mount', '-o', 'loop', + location, mount_point]) except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) @@ -287,7 +359,7 @@ class WriteExtension(cliapp.Application): yield mount_point finally: self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) + subprocess.check_call(['umount', mount_point]) os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, @@ -334,9 +406,9 @@ class WriteExtension(cliapp.Application): orig = os.path.join(version_root, 'orig') self.status(msg='Creating orig subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + subprocess.check_call(['btrfs', 'subvolume', 'create', orig]) self.status(msg='Copying files to orig subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) + subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.']) def create_run(self, version_root): '''Create the 'run' snapshot.''' @@ -344,7 +416,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating run subvolume') orig = os.path.join(version_root, 'orig') run = os.path.join(version_root, 'run') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', orig, run]) def create_state_subvolume(self, system_dir, mountpoint, state_subdir): @@ -358,7 +430,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume]) os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) @@ -369,7 +441,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Moving existing data to %s subvolume' % subvolume) for filename in files: filepath = os.path.join(existing_state_dir, filename) - cliapp.runcmd(['mv', filepath, subvolume]) + subprocess.check_call(['mv', filepath, subvolume]) def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. @@ -419,7 +491,7 @@ class WriteExtension(cliapp.Application): if 'INITRAMFS_PATH' in os.environ: initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) if not os.path.exists(initramfs): - raise morphlib.Error('INITRAMFS_PATH specified, ' + raise ExtensionError('INITRAMFS_PATH specified, ' 'but file does not exist') return initramfs return None @@ -432,7 +504,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Installing initramfs') initramfs_dest = os.path.join(version_root, 'initramfs') - cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest]) def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -443,7 +515,7 @@ class WriteExtension(cliapp.Application): for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + subprocess.check_call(['cp', '-a', try_path, kernel_dest]) break def install_dtb(self, version_root, temp_root): @@ -454,10 +526,10 @@ class WriteExtension(cliapp.Application): dtb_dest = os.path.join(version_root, 'dtb') try_path = os.path.join(temp_root, device_tree_path) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + subprocess.check_call(['cp', '-a', try_path, dtb_dest]) else: logging.error("Failed to find device tree %s", device_tree_path) - raise cliapp.AppException( + raise ExtensionError( 'Failed to find device tree %s' % device_tree_path) def get_dtb_path(self): @@ -489,7 +561,7 @@ class WriteExtension(cliapp.Application): if config_type in config_function_dict: config_function_dict[config_type](real_root, disk_uuid) else: - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) def generate_extlinux_config(self, real_root, disk_uuid=None): @@ -533,15 +605,15 @@ class WriteExtension(cliapp.Application): if install_type in install_function_dict: install_function_dict[install_type](real_root) elif install_type != 'none': - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_INSTALL %s' % install_type) def install_bootloader_extlinux(self, real_root): self.status(msg='Installing extlinux') - cliapp.runcmd(['extlinux', '--install', real_root]) + subprocess.check_call(['extlinux', '--install', real_root]) # FIXME this hack seems to be necessary to let extlinux finish - cliapp.runcmd(['sync']) + subprocess.check_call(['sync']) time.sleep(2) def install_syslinux_menu(self, real_root, version_root): @@ -599,19 +671,19 @@ class WriteExtension(cliapp.Application): elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for %s: %s' % - (variable, value)) + raise ExtensionError('Unexpected value for %s: %s' % + (variable, value)) def check_ssh_connectivity(self, ssh_host): try: - output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) - except cliapp.AppException as e: + output = run_ssh_command(ssh_host, ['echo', 'test']) + except subprocess.CalledProcessError as e: logging.error("Error checking SSH connectivity: %s", str(e)) - raise cliapp.AppException( + raise ExtensionError( 'Unable to SSH to %s: %s' % (ssh_host, e)) if output.strip() != 'test': - raise cliapp.AppException( + raise ExtensionError( 'Unexpected output from remote machine: %s' % output.strip()) def is_device(self, location): -- cgit v1.2.1