From d8640b08d111c934596f861f69cd6503bee32c36 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 24 Jun 2013 17:08:12 +0000 Subject: Remove duplicate copies of deployment extensions --- add-config-files.configure | 27 ----- install-files.configure | 112 --------------------- kvm.write | 128 ----------------------- nfsboot.configure | 32 ------ nfsboot.write | 245 --------------------------------------------- rawdisk.write | 104 ------------------- set-hostname.configure | 27 ----- simple-network.configure | 143 -------------------------- ssh-rsync.write | 193 ----------------------------------- ssh.configure | 162 ------------------------------ tar.write | 21 ---- virtualbox-ssh.write | 206 ------------------------------------- 12 files changed, 1400 deletions(-) delete mode 100755 add-config-files.configure delete mode 100755 install-files.configure delete mode 100755 kvm.write delete mode 100755 nfsboot.configure delete mode 100755 nfsboot.write delete mode 100755 rawdisk.write delete mode 100755 set-hostname.configure delete mode 100755 simple-network.configure delete mode 100755 ssh-rsync.write delete mode 100755 ssh.configure delete mode 100755 tar.write delete mode 100755 virtualbox-ssh.write diff --git a/add-config-files.configure b/add-config-files.configure deleted file mode 100755 index 0094cf6..0000000 --- a/add-config-files.configure +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Copy all files located in $SRC_CONFIG_DIR to the image /etc. - - -set -e - -if [ "x${SRC_CONFIG_DIR}" != x ] -then - cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" -fi - diff --git a/install-files.configure b/install-files.configure deleted file mode 100755 index 669fc51..0000000 --- a/install-files.configure +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -''' A Morph configuration extension for adding arbitrary files to a system - -It will read the manifest files specified in the environment variable -INSTALL_FILES, then use the contens of those files to determine which files -to install into the target system. - -''' - -import cliapp -import os -import re -import sys -import shlex -import shutil -import stat - -class InstallFilesConfigureExtension(cliapp.Application): - - '''Install the files specified in the manifests listed in INSTALL_FILES - - The manifest is formatted as: - - - - Where the filename is how the file is found inside whatever directory - the manifest is stored in, and also the path within the system to - install to. - - Directories on the target must be created if they do not exist. - - This extension supports files, symlinks and directories. - - ''' - - def process_args(self, args): - if not 'INSTALL_FILES' in os.environ: - return - target_root = args[0] - manifests = shlex.split(os.environ['INSTALL_FILES']) - for manifest in manifests: - self.install_manifest(manifest, target_root) - - def install_manifest(self, manifest, target_root): - manifest_dir = os.path.dirname(manifest) - with open(manifest) as f: - entries = f.readlines() - for entry in entries: - self.install_entry(entry, manifest_dir, target_root) - - def install_entry(self, entry, manifest_root, target_root): - entry_data = re.split('\W+', entry.strip(), maxsplit=3) - mode = int(entry_data[0], 8) - uid = int(entry_data[1]) - gid = int(entry_data[2]) - path = entry_data[3] - dest_path = os.path.join(target_root, './' + path) - if stat.S_ISDIR(mode): - if os.path.exists(dest_path): - dest_stat = os.stat(dest_path) - if (mode != dest_stat.st_mode - or uid != dest_stat.st_uid - or gid != dest_stat.st_gid): - raise cliapp.AppException('"%s" exists and is not ' - 'identical to directory ' - '"%s"' % (dest_path, entry)) - else: - os.mkdir(dest_path, mode) - os.chown(dest_path, uid, gid) - os.chmod(dest_path, mode) - - elif stat.S_ISLNK(mode): - if os.path.lexists(dest_path): - raise cliapp.AppException('Symlink already exists at %s' - % dest_path) - else: - linkdest = os.readlink(os.path.join(manifest_root, - './' + path)) - os.symlink(linkdest, dest_path) - os.lchown(dest_path, uid, gid) - - elif stat.S_ISREG(mode): - if os.path.lexists(dest_path): - raise cliapp.AppException('File already exists at %s' - % dest_path) - else: - shutil.copyfile(os.path.join(manifest_root, './' + path), - dest_path) - os.chown(dest_path, uid, gid) - os.chmod(dest_path, mode) - - else: - raise cliapp.AppException('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) - -InstallFilesConfigureExtension().run() diff --git a/kvm.write b/kvm.write deleted file mode 100755 index 67ac40e..0000000 --- a/kvm.write +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''A Morph deployment write extension for deploying to KVM+libvirt.''' - - -import cliapp -import os -import re -import sys -import tempfile -import urlparse - -import morphlib.writeexts - - -class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - - '''Create a KVM/LibVirt virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - kvm+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which KVM/LibVirt is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run libvirt's - command line management tools. - - ''' - - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.parse_autostart() - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - - try: - self.transfer(raw_disk, ssh_host, vm_path) - self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) - except BaseException: - sys.stderr.write('Error deploying to libvirt') - os.remove(raw_disk) - raise - else: - os.remove(raw_disk) - - self.status( - msg='Virtual machine %(vm_name)s has been created', - vm_name=vm_name) - - def parse_location(self, location): - '''Parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - if x.scheme != 'kvm+ssh': - raise cliapp.AppException( - 'URL schema must be vbox+ssh in %s' % location) - m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) - if not m: - raise cliapp.AppException('Cannot parse location %s' % location) - return x.netloc, m.group('guest'), m.group('path') - - def transfer(self, raw_disk, ssh_host, vm_path): - '''Transfer raw disk image to libvirt host.''' - - self.status(msg='Transferring disk image') - target = '%s:%s' % (ssh_host, vm_path) - with open(raw_disk, 'rb') as f: - cliapp.runcmd(['rsync', '-szS', raw_disk, target]) - - def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): - '''Create the libvirt virtual machine.''' - - self.status(msg='Creating libvirt/kvm virtual machine') - - attach_disks = self.parse_attach_disks() - attach_opts = [] - for disk in attach_disks: - attach_opts.extend(['--disk', 'path=%s' % disk]) - - if 'NIC_CONFIG' in os.environ: - nics = os.environ['NIC_CONFIG'].split() - for nic in nics: - attach_opts.extend(['--network', nic]) - - ram_mebibytes = str(self.get_ram_size() / (1024**2)) - - cmdline = ['virt-install', '--connect', 'qemu:///system', '--import', - '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, - '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts - if not autostart: - cmdline += ['--noreboot'] - cliapp.ssh_runcmd(ssh_host, cmdline) - - if autostart: - cliapp.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) - -KvmPlusSshWriteExtension().run() - diff --git a/nfsboot.configure b/nfsboot.configure deleted file mode 100755 index 8dc6c67..0000000 --- a/nfsboot.configure +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Remove all networking interfaces and stop fstab from mounting '/' - - -set -e -if [ "$NFSBOOT_CONFIGURE" ]; then - # Remove all networking interfaces but loopback - cat > "$1/etc/network/interfaces" < "$1/etc/fstab" -fi diff --git a/nfsboot.write b/nfsboot.write deleted file mode 100755 index 34a7297..0000000 --- a/nfsboot.write +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''A Morph deployment write extension for deploying to an nfsboot server - -An nfsboot server is defined as a baserock system that has tftp and nfs -servers running, the tftp server is exporting the contents of -/srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots -in /srv/nfsboot/nfs/ - -''' - - -import cliapp -import os -import glob - -import morphlib.writeexts - - -class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): - - '''Create an NFS root and kernel on TFTP during Morph's deployment. - - The location command line argument is the hostname of the nfsboot server. - The user is expected to provide the location argument - using the following syntax: - - HOST - - where: - - * HOST is the host of the nfsboot server - - The extension will connect to root@HOST via ssh to copy the kernel and - rootfs, and configure the nfs server. - - It requires root because it uses systemd, and reads/writes to /etc. - - ''' - - _nfsboot_root = '/srv/nfsboot' - - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - hostname = self.get_hostname(temp_root) - if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "baserock"') - - self.test_good_server(location) - version_label = os.getenv('VERSION_LABEL', 'factory') - versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', - version_label) - if self.version_exists(versioned_root, location): - raise cliapp.AppException('Version %s already exists on' - ' this device. Deployment aborted' - % version_label) - self.copy_rootfs(temp_root, location, versioned_root, hostname) - self.copy_kernel(temp_root, location, versioned_root, version_label, - hostname) - self.configure_nfs(location, hostname) - - def version_exists(self, versioned_root, location): - try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: - return False - - return True - - def get_hostname(self, temp_root): - hostnamepath = os.path.join(temp_root, 'etc', 'hostname') - with open(hostnamepath) as f: - return f.readline().strip() - - def create_local_state(self, location, hostname): - statedir = os.path.join(self._nfsboot_root, hostname, 'state') - subdirs = [os.path.join(statedir, 'home'), - os.path.join(statedir, 'opt'), - os.path.join(statedir, 'srv')] - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) - - def copy_kernel(self, temp_root, location, versioned_root, version, - hostname): - bootdir = os.path.join(temp_root, 'boot') - image_names = ['vmlinuz', 'zImage', 'uImage'] - for name in image_names: - try_path = os.path.join(bootdir, name) - if os.path.exists(try_path): - kernel_src = try_path - break - else: - raise cliapp.AppException( - 'Could not find a kernel in the system: none of ' - '%s found' % ', '.join(image_names)) - - kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') - rsync_dest = 'root@%s:%s' % (location, kernel_dest) - self.status(msg='Copying kernel') - cliapp.runcmd( - ['rsync', '-s', kernel_src, rsync_dest]) - - # Link the kernel to the right place - self.status(msg='Creating links to kernel in tftp directory') - tftp_dir = os.path.join(self._nfsboot_root , 'tftp') - versioned_kernel_name = "%s-%s" % (hostname, version) - kernel_name = hostname - try: - cliapp.ssh_runcmd('root@%s' % location, - ['ln', '-f', kernel_dest, - os.path.join(tftp_dir, versioned_kernel_name)]) - - cliapp.ssh_runcmd('root@%s' % location, - ['ln', '-sf', versioned_kernel_name, - os.path.join(tftp_dir, kernel_name)]) - except cliapp.AppException: - raise cliapp.AppException('Could not create symlinks to the ' - 'kernel at %s in %s on %s' - % (kernel_dest, tftp_dir, location)) - - def copy_rootfs(self, temp_root, location, versioned_root, hostname): - rootfs_src = temp_root + '/' - orig_path = os.path.join(versioned_root, 'orig') - run_path = os.path.join(versioned_root, 'run') - - self.status(msg='Creating destination directories') - try: - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except cliapp.AppException: - raise cliapp.AppException('Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) - - self.status(msg='Creating \'orig\' rootfs') - cliapp.runcmd( - ['rsync', '-asXSPH', '--delete', rootfs_src, - 'root@%s:%s' % (location, orig_path)]) - - self.status(msg='Creating \'run\' rootfs') - try: - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', os.path.join(run_path, 'etc')]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-a', os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except cliapp.AppException: - raise cliapp.AppException('Could not create \'run\' rootfs' - ' from \'orig\'') - - self.status(msg='Linking \'default\' to latest system') - try: - cliapp.ssh_runcmd('root@%s' % location, - ['ln', '-sfn', versioned_root, - os.path.join(self._nfsboot_root, hostname, 'systems', - 'default')]) - except cliapp.AppException: - raise cliapp.AppException('Could not link \'default\' to %s' - % versioned_root) - - def configure_nfs(self, location, hostname): - exported_path = os.path.join(self._nfsboot_root, hostname) - exports_path = '/etc/exports' - # If that path is not already exported: - try: - cliapp.ssh_runcmd( - 'root@%s' % location, ['grep', '-q', exported_path, - exports_path]) - except cliapp.AppException: - ip_mask = '*' - options = 'rw,no_subtree_check,no_root_squash,async' - exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) - exports_append_sh = '''\ -set -eu -target="$1" -temp=$(mktemp) -cat "$target" > "$temp" -cat >> "$temp" -mv "$temp" "$target" -''' - cliapp.ssh_runcmd( - 'root@%s' % location, - ['sh', '-c', exports_append_sh, '--', exports_path], - feed_stdin=exports_string) - cliapp.ssh_runcmd( - 'root@%s' % location, ['systemctl', 'restart', - 'nfs-server.service']) - - def test_good_server(self, server): - # Can be ssh'ed into - try: - cliapp.ssh_runcmd('root@%s' % server, ['true']) - except cliapp.AppException: - raise cliapp.AppException('You are unable to ssh into server %s' - % server) - - # Is an NFS server - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['systemctl', 'is-enabled', - 'nfs-server.service']) - - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) - - # TFTP server exports /srv/nfsboot/tftp - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export ' - '/srv/nfsboot/tftp' % server) - -NFSBootWriteExtension().run() - diff --git a/rawdisk.write b/rawdisk.write deleted file mode 100755 index a74d690..0000000 --- a/rawdisk.write +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''A Morph deployment write extension for raw disk images.''' - - -import cliapp -import os -import sys -import time -import tempfile - -import morphlib.writeexts - - -class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): - - '''Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - - The location command line argument is the pathname of the disk image - to be created/upgraded. - - ''' - - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - if os.path.isfile(location): - self.upgrade_local_system(location, temp_root) - else: - self.create_local_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % location) - - def upgrade_local_system(self, raw_disk, temp_root): - mp = self.mount(raw_disk) - - version_label = self.get_version_label(mp) - self.status(msg='Updating image to a new version with label %s' % - version_label) - - version_root = os.path.join(mp, 'systems', version_label) - os.mkdir(version_root) - - old_orig = os.path.join(mp, 'systems', 'factory', 'orig') - new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( - ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - - cliapp.runcmd( - ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', - temp_root + os.path.sep, new_orig]) - - self.create_run(version_root) - - default_path = os.path.join(mp, 'systems', 'default') - if os.path.exists(default_path): - os.remove(default_path) - else: - # we are upgrading and old system that does - # not have an updated extlinux config file - if self.bootloader_is_wanted(): - self.install_extlinux(mp) - os.symlink(version_label, default_path) - - if self.bootloader_is_wanted(): - self.install_kernel(version_root, temp_root) - - self.unmount(mp) - - def get_version_label(self, mp): - version_label = os.environ.get('VERSION_LABEL') - - if version_label is None: - self.unmount(mp) - raise cliapp.AppException('VERSION_LABEL was not given') - - if os.path.exists(os.path.join(mp, 'systems', version_label)): - self.unmount(mp) - raise cliapp.AppException('VERSION_LABEL %s already exists' - % version_label) - - return version_label - - -RawDiskWriteExtension().run() - diff --git a/set-hostname.configure b/set-hostname.configure deleted file mode 100755 index e44c5d5..0000000 --- a/set-hostname.configure +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Set hostname on system from HOSTNAME. - - -set -e - -if [ -n "$HOSTNAME" ] -then - echo "$HOSTNAME" > "$1/etc/hostname" -fi - diff --git a/simple-network.configure b/simple-network.configure deleted file mode 100755 index b98b202..0000000 --- a/simple-network.configure +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -'''A Morph deployment configuration extension to handle /etc/network/interfaces - -This extension prepares /etc/network/interfaces with the interfaces specified -during deployment. - -If no network configuration is provided, eth0 will be configured for DHCP -with the hostname of the system. -''' - - -import os -import sys -import cliapp - -import morphlib - - -class SimpleNetworkError(morphlib.Error): - '''Errors associated with simple network setup''' - pass - - -class SimpleNetworkConfigurationExtension(cliapp.Application): - '''Configure /etc/network/interfaces - - Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces. - ''' - - def process_args(self, args): - network_config = os.environ.get( - "NETWORK_CONFIG", "lo:loopback;eth0:dhcp,hostname=$(hostname)") - - self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) - - stanzas = self.parse_network_stanzas(network_config) - iface_file = self.generate_iface_file(stanzas) - - with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: - f.write(iface_file) - - def generate_iface_file(self, stanzas): - """Generate an interfaces file from the provided stanzas. - - The interfaces will be sorted by name, with loopback sorted first. - """ - - def cmp_iface_names(a, b): - a = a['name'] - b = b['name'] - if a == "lo": - return -1 - elif b == "lo": - return 1 - else: - return cmp(a,b) - - return "\n".join(self.generate_iface_stanza(stanza) - for stanza in sorted(stanzas, cmp=cmp_iface_names)) - - def generate_iface_stanza(self, stanza): - """Generate an interfaces stanza from the provided data.""" - - name = stanza['name'] - itype = stanza['type'] - lines = ["auto %s" % name, "iface %s inet %s" % (name, itype)] - lines += [" %s %s" % elem for elem in stanza['args'].items()] - lines += [""] - return "\n".join(lines) - - - def parse_network_stanzas(self, config): - """Parse a network config environment variable into stanzas. - - Network config stanzas are semi-colon separated. - """ - - return [self.parse_network_stanza(s) for s in config.split(";")] - - def parse_network_stanza(self, stanza): - """Parse a network config stanza into name, type and arguments. - - Each stanza is of the form name:type[,arg=value]... - - For example: - lo:loopback - eth0:dhcp - eth1:static,address=10.0.0.1,netmask=255.255.0.0 - """ - elements = stanza.split(",") - lead = elements.pop(0).split(":") - if len(lead) != 2: - raise SimpleNetworkError("Stanza '%s' is missing its type" % - stanza) - iface = lead[0] - iface_type = lead[1] - - if iface_type not in ['loopback', 'static', 'dhcp']: - raise SimpleNetworkError("Stanza '%s' has unknown interface type" - " '%s'" % (stanza, iface_type)) - - argpairs = [element.split("=", 1) for element in elements] - output_stanza = { "name": iface, - "type": iface_type, - "args": {} } - for argpair in argpairs: - if len(argpair) != 2: - raise SimpleNetworkError("Stanza '%s' has bad argument '%r'" - % (stanza, argpair.pop(0))) - if argpair[0] in output_stanza["args"]: - raise SimpleNetworkError("Stanza '%s' has repeated argument" - " %s" % (stanza, argpair[0])) - output_stanza["args"][argpair[0]] = argpair[1] - - return output_stanza - - def status(self, **kwargs): - '''Provide status output. - - The ``msg`` keyword argument is the actual message, - the rest are values for fields in the message as interpolated - by %. - - ''' - - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - -SimpleNetworkConfigurationExtension().run() diff --git a/ssh-rsync.write b/ssh-rsync.write deleted file mode 100755 index fba550c..0000000 --- a/ssh-rsync.write +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''A Morph deployment write extension for upgrading systems over ssh.''' - - -import cliapp -import os -import sys -import time -import tempfile - -import morphlib.writeexts - -class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): - - '''Upgrade a running baserock system with ssh and rsync. - - It assumes the system is baserock-based and has a btrfs partition. - - The location command line argument is the 'user@hostname' string - that will be passed to ssh and rsync - - ''' - - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - - self.check_valid_target(location) - self.upgrade_remote_system(location, temp_root) - - def upgrade_remote_system(self, location, temp_root): - root_disk = self.find_root_disk(location) - version_label = os.environ.get('VERSION_LABEL') - - try: - self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() - - self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) - - version_root = os.path.join(remote_mnt, 'systems', version_label) - run_dir = os.path.join(version_root, 'run') - orig_dir = os.path.join(version_root, 'orig') - try: - self.status(msg='Creating %s' % version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) - - self.create_remote_orig(location, version_root, remote_mnt, - temp_root) - - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1"' - ' || ' - 'echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-s', '-f', - version_label, - default_path]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass - raise e - - except: - raise - else: - self.status(msg='Removing temporary mounts') - cliapp.ssh_runcmd(location, ['umount', root_disk]) - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - - def update_remote_extlinux(self, location, remote_mnt, version_label): - '''Install/reconfigure extlinux on location''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(remote_mnt, 'extlinux.conf') - temp_file = tempfile.mkstemp()[1] - with open(temp_file, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') - - cliapp.ssh_runcmd(location, ['mv', config, config+'~']) - - try: - cliapp.runcmd(['rsync', '-as', temp_file, - '%s:%s' % (location, config)]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) - except: - pass - raise e - - def create_remote_orig(self, location, version_root, remote_mnt, - temp_root): - '''Create the subvolume version_root/orig on location''' - - self.status(msg='Creating "orig" subvolume') - old_orig = self.get_old_orig(location, remote_mnt) - new_orig = os.path.join(version_root, 'orig') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', - old_orig, new_orig]) - - cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root, '%s:%s' % (location, new_orig)]) - - def get_old_orig(self, location, remote_mnt): - '''Identify which subvolume to snapshot from''' - - # rawdisk upgrades use 'factory' - return os.path.join(remote_mnt, 'systems', 'factory', 'orig') - - def find_root_disk(self, location): - '''Read /proc/mounts on location to find which device contains "/"''' - - self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) - for line in contents.splitlines(): - line_words = line.split() - if (line_words[1] == '/' and line_words[0] != 'rootfs'): - return line_words[0] - - def install_remote_kernel(self, location, version_root, temp_root): - '''Install the kernel in temp_root inside version_root on location''' - - self.status(msg='Installing kernel') - image_names = ['vmlinuz', 'zImage', 'uImage'] - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) - - def check_valid_target(self, location): - try: - cliapp.ssh_runcmd(location, ['true']) - except Exception as e: - raise cliapp.AppException('%s does not respond to ssh:\n%s' - % (location, e)) - - try: - cliapp.ssh_runcmd(location, ['test', '-d', '/baserock']) - except: - raise cliapp.AppException('%s is not a baserock system' % location) - - try: - cliapp.ssh_runcmd(location, ['which', 'rsync']) - except: - raise cliapp.AppException('%s does not have rsync') - -SshRsyncWriteExtension().run() diff --git a/ssh.configure b/ssh.configure deleted file mode 100755 index 2f3167e..0000000 --- a/ssh.configure +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -'''A Morph deployment configuration to copy SSH keys. - -Keys are copied from the host to the new system. -''' - -import cliapp -import os -import sys -import shutil -import glob -import re -import logging - -import morphlib - -class SshConfigurationExtension(cliapp.Application): - - '''Copy over SSH keys to new system from host. - - The extension requires SSH_KEY_DIR to be set at the command line as it - will otherwise pass with only a status update. SSH_KEY_DIR should be - set to the location of the SSH keys to be passed to the new system. - - ''' - - def process_args(self, args): - if 'SSH_KEY_DIR' in os.environ: - # Copies ssh_host keys. - key = 'ssh_host_*_key' - mode = 0755 - dest = os.path.join(args[0], 'etc/ssh/') - sshhost, sshhostpub = self.find_keys(key) - if sshhost or sshhostpub: - self.check_dir(dest, mode) - self.copy_keys(sshhost, sshhostpub, dest) - - # Copies root keys. - key = 'root_*_key' - mode = 0700 - dest = os.path.join(args[0], 'root/.ssh/') - roothost, roothostpub = self.find_keys(key) - key = 'root_authorized_key_*.pub' - authkey, bleh = self.find_keys(key) - if roothost or roothostpub: - self.check_dir(dest, mode) - self.copy_rename_keys(roothost, - roothostpub, dest, 'id_', [5, 4]) - if authkey: - self.check_dir(dest, mode) - self.comb_auth_key(authkey, dest) - - # Fills the known_hosts file - key = 'root_known_host_*_key.pub' - src = os.path.join(os.environ['SSH_KEY_DIR'], key) - known_hosts_keys = glob.glob(src) - if known_hosts_keys: - self.check_dir(dest, mode) - known_hosts_path = os.path.join(dest, 'known_hosts') - with open(known_hosts_path, "a") as known_hosts_file: - for filename in known_hosts_keys: - hostname = re.search('root_known_host_(.+?)_key.pub', - filename).group(1) - known_hosts_file.write(hostname + " ") - with open(filename, "r") as f: - shutil.copyfileobj(f, known_hosts_file) - - else: - self.status(msg="No SSH key directory found.") - pass - - def find_keys(self, key_name): - '''Uses glob to find public and - private SSH keys and returns their path''' - - src = os.path.join(os.environ['SSH_KEY_DIR'], key_name) - keys = glob.glob(src) - pubkeys = glob.glob(src + '.pub') - if not (keys or pubkeys): - self.status(msg="No SSH keys of pattern %(src)s found.", src=src) - return keys, pubkeys - - def check_dir(self, dest, mode): - '''Checks if destination directory exists - and creates it if necessary''' - - if os.path.exists(dest) == False: - self.status(msg="Creating SSH key directory: %(dest)s", dest=dest) - os.mkdir(dest) - os.chmod(dest, mode) - else: - pass - - def copy_keys(self, keys, pubkeys, dest): - '''Copies SSH keys to new VM''' - - for key in keys: - shutil.copy(key, dest) - path = os.path.join(dest, os.path.basename(key)) - os.chmod(path, 0600) - for key in pubkeys: - shutil.copy(key, dest) - path = os.path.join(dest, os.path.basename(key)) - os.chmod(path, 0644) - - def copy_rename_keys(self, keys, pubkeys, dest, new, snip): - '''Copies SSH keys to new VM and renames them''' - - st, fi = snip - for key in keys: - base = os.path.basename(key) - s = len(base) - nw_dst = os.path.join(dest, new + base[st:s-fi]) - shutil.copy(key, nw_dst) - os.chmod(nw_dst, 0600) - for key in pubkeys: - base = os.path.basename(key) - s = len(base) - nw_dst = os.path.join(dest, new + base[st:s-fi-4]) - shutil.copy(key, nw_dst + '.pub') - os.chmod(nw_dst + '.pub', 0644) - - def comb_auth_key(self, keys, dest): - '''Combines authorized_keys file in new VM''' - - dest = os.path.join(dest, 'authorized_keys') - fout = open(dest, 'a') - for key in keys: - fin = open(key, 'r') - data = fin.read() - fout.write(data) - fin.close() - fout.close() - os.chmod(dest, 0600) - - def status(self, **kwargs): - '''Provide status output. - - The ``msg`` keyword argument is the actual message, - the rest are values for fields in the message as interpolated - by %. - - ''' - - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - -SshConfigurationExtension().run() diff --git a/tar.write b/tar.write deleted file mode 100755 index 333626b..0000000 --- a/tar.write +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# A Morph write extension to deploy to a .tar file - -set -eu - -tar -C "$1" -cf "$2" . diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write deleted file mode 100755 index 3ee2eae..0000000 --- a/virtualbox-ssh.write +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''A Morph deployment write extension for deploying to VirtualBox via ssh. - -VirtualBox is assumed to be running on a remote machine, which is -accessed over ssh. The machine gets created, but not started. - -''' - - -import cliapp -import os -import re -import sys -import time -import tempfile -import urlparse - -import morphlib.writeexts - - -class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - - '''Create a VirtualBox virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - vbox+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which VirtualBox is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run VirtualBox's - command line management tools. - - ''' - - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.parse_autostart() - - fd, raw_disk = tempfile.mkstemp() - os.close(fd) - self.create_local_system(temp_root, raw_disk) - - try: - self.transfer_and_convert_to_vdi( - raw_disk, ssh_host, vdi_path) - self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, - autostart) - except BaseException: - sys.stderr.write('Error deploying to VirtualBox') - os.remove(raw_disk) - raise - else: - os.remove(raw_disk) - - self.status( - msg='Virtual machine %(vm_name)s has been created', - vm_name=vm_name) - - def parse_location(self, location): - '''Parse the location argument to get relevant data.''' - - x = urlparse.urlparse(location) - if x.scheme != 'vbox+ssh': - raise cliapp.AppException( - 'URL schema must be vbox+ssh in %s' % location) - m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) - if not m: - raise cliapp.AppException('Cannot parse location %s' % location) - return x.netloc, m.group('guest'), m.group('path') - - def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): - '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' - - self.status(msg='Transfer disk and convert to VDI') - with open(raw_disk, 'rb') as f: - cliapp.ssh_runcmd(ssh_host, - ['VBoxManage', 'convertfromraw', 'stdin', vdi_path, - str(os.path.getsize(raw_disk))], - stdin=f) - - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): - '''Create the VirtualBox virtual machine.''' - - self.status(msg='Create VirtualBox virtual machine') - - ram_mebibytes = str(self.get_ram_size() / (1024**2)) - - hostonly_iface = self.get_host_interface(ssh_host) - - commands = [ - ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', - '--register'], - ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, - '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, - '--nic2', 'nat', '--natnet2', 'default'], - ['storagectl', vm_name, '--name', '"SATA Controller"', - '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], - ['storageattach', vm_name, '--storagectl', '"SATA Controller"', - '--port', '0', '--device', '0', '--type', 'hdd', '--medium', - vdi_path], - ] - - attach_disks = self.parse_attach_disks() - for device_no, disk in enumerate(attach_disks, 1): - cmd = ['storageattach', vm_name, - '--storagectl', '"SATA Controller"', - '--port', str(device_no), - '--device', '0', - '--type', 'hdd', - '--medium', disk] - commands.append(cmd) - - if autostart: - commands.append(['startvm', vm_name]) - - for command in commands: - argv = ['VBoxManage'] + command - cliapp.ssh_runcmd(ssh_host, argv) - - def get_host_interface(self, ssh_host): - host_ipaddr = os.environ.get('HOST_IPADDR') - netmask = os.environ.get('NETMASK') - network_config = os.environ.get("NETWORK_CONFIG") - - if network_config is None: - raise cliapp.AppException('NETWORK_CONFIG was not given') - - if "eth0:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth0 configuration') - - if "eth1:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth1 configuration') - - if host_ipaddr is None: - raise cliapp.AppException('HOST_IPADDR was not given') - - if netmask is None: - raise cliapp.AppException('NETMASK was not given') - - # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly - # interfaces on the host. For each interface, the following lines - # are shown on top: - # - # Name: vboxnet0 - # GUID: 786f6276-656e-4074-8000-0a0027000000 - # Dhcp: Disabled - # IPAddress: 192.168.100.1 - # - # The following command tries to retrieve the hostonly interface - # name (e.g. vboxnet0) associated with the given ip address. - iface = None - lines = cliapp.ssh_runcmd(ssh_host, - ['VBoxManage', 'list', 'hostonlyifs']).splitlines() - for i, v in enumerate(lines): - if host_ipaddr in v: - iface = lines[i-3].split()[1] - break - - if iface is None: - iface = cliapp.ssh_runcmd(ssh_host, - ['VBoxManage', 'hostonlyif', 'create']) - # 'VBoxManage hostonlyif create' shows the name of the - # created hostonly interface inside single quotes - iface = iface[iface.find("'") + 1 : iface.rfind("'")] - cliapp.ssh_runcmd(ssh_host, - ['VBoxManage', 'hostonlyif', - 'ipconfig', iface, - '--ip', host_ipaddr, - '--netmask', netmask]) - - return iface - -VirtualBoxPlusSshWriteExtension().run() - -- cgit v1.2.1