diff options
-rwxr-xr-x | kvm.write | 121 | ||||
-rwxr-xr-x | nfsboot.configure | 32 | ||||
-rwxr-xr-x | nfsboot.write | 245 | ||||
-rwxr-xr-x | rawdisk.write | 95 | ||||
-rwxr-xr-x | set-hostname.configure | 27 | ||||
-rwxr-xr-x | simple-network.configure | 143 | ||||
-rwxr-xr-x | ssh-rsync.write | 181 | ||||
-rwxr-xr-x | ssh.configure | 162 | ||||
-rwxr-xr-x | tar.write | 21 | ||||
-rwxr-xr-x | virtualbox-ssh.write | 148 |
10 files changed, 1175 insertions, 0 deletions
diff --git a/kvm.write b/kvm.write new file mode 100755 index 0000000..ae287fe --- /dev/null +++ b/kvm.write @@ -0,0 +1,121 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to KVM+libvirt.''' + + +import cliapp +import os +import re +import sys +import tempfile +import urlparse + +import morphlib.writeexts + + +class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a KVM/LibVirt virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + kvm+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which KVM/LibVirt is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run libvirt's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vm_path = self.parse_location(location) + autostart = self.parse_autostart() + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + + try: + self.transfer(raw_disk, ssh_host, vm_path) + self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) + except BaseException: + sys.stderr.write('Error deploying to libvirt') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'kvm+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer(self, raw_disk, ssh_host, vm_path): + '''Transfer raw disk image to libvirt host.''' + + self.status(msg='Transferring disk image') + target = '%s:%s' % (ssh_host, vm_path) + with open(raw_disk, 'rb') as f: + cliapp.runcmd(['rsync', '-zS', raw_disk, target]) + + def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): + '''Create the libvirt virtual machine.''' + + self.status(msg='Creating libvirt/kvm virtual machine') + + attach_disks = self.parse_attach_disks() + attach_opts = [] + for disk in attach_disks: + attach_opts.extend(['--disk', 'path=%s' % disk]) + + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + + cmdline = ['ssh', ssh_host, + 'virt-install', '--connect qemu:///system', '--import', + '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, + '--disk path=%s,bus=ide' % vm_path] + attach_opts + if not autostart: + cmdline += ['--noreboot'] + cliapp.runcmd(cmdline) + + +KvmPlusSshWriteExtension().run() + diff --git a/nfsboot.configure b/nfsboot.configure new file mode 100755 index 0000000..8dc6c67 --- /dev/null +++ b/nfsboot.configure @@ -0,0 +1,32 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Remove all networking interfaces and stop fstab from mounting '/' + + +set -e +if [ "$NFSBOOT_CONFIGURE" ]; then + # Remove all networking interfaces but loopback + cat > "$1/etc/network/interfaces" <<EOF +auto lo +iface lo inet loopback +EOF + + # Stop fstab from mounting '/' + mv "$1/etc/fstab" "$1/etc/fstab.old" + awk '/^ *#/ || $2 != "/"' "$1/etc/fstab.old" > "$1/etc/fstab" +fi diff --git a/nfsboot.write b/nfsboot.write new file mode 100755 index 0000000..61c5306 --- /dev/null +++ b/nfsboot.write @@ -0,0 +1,245 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to an nfsboot server + +An nfsboot server is defined as a baserock system that has tftp and nfs +servers running, the tftp server is exporting the contents of +/srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots +in /srv/nfsboot/nfs/ + +''' + + +import cliapp +import os +import glob + +import morphlib.writeexts + + +class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create an NFS root and kernel on TFTP during Morph's deployment. + + The location command line argument is the hostname of the nfsboot server. + The user is expected to provide the location argument + using the following syntax: + + HOST + + where: + + * HOST is the host of the nfsboot server + + The extension will connect to root@HOST via ssh to copy the kernel and + rootfs, and configure the nfs server. + + It requires root because it uses systemd, and reads/writes to /etc. + + ''' + + _nfsboot_root = '/srv/nfsboot' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + hostname = self.get_hostname(temp_root) + if hostname == 'baserock': + raise cliapp.AppException('It is forbidden to nfsboot a system ' + 'with hostname "baserock"') + + self.test_good_server(location) + version = os.environ['VERSION'] or 'version1' + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version) + if self.version_exists(versioned_root, location): + raise cliapp.AppException('Version %s already exists on' + ' this device. Deployment aborted' + % version) + self.copy_rootfs(temp_root, location, versioned_root, hostname) + self.copy_kernel(temp_root, location, versioned_root, version, + hostname) + self.configure_nfs(location, hostname) + + def version_exists(self, versioned_root, location): + try: + cliapp.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except cliapp.AppException: + return False + + return True + + def get_hostname(self, temp_root): + hostnamepath = os.path.join(temp_root, 'etc', 'hostname') + with open(hostnamepath) as f: + return f.readline().strip() + + def create_local_state(self, location, hostname): + statedir = os.path.join(self._nfsboot_root, hostname, 'state') + subdirs = [os.path.join(statedir, 'home'), + os.path.join(statedir, 'opt'), + os.path.join(statedir, 'srv')] + cliapp.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) + + def copy_kernel(self, temp_root, location, versioned_root, version, + hostname): + bootdir = os.path.join(temp_root, 'boot') + image_names = ['vmlinuz', 'zImage', 'uImage'] + for name in image_names: + try_path = os.path.join(bootdir, name) + if os.path.exists(try_path): + kernel_src = try_path + break + else: + raise cliapp.AppException( + 'Could not find a kernel in the system: none of ' + '%s found' % ', '.join(image_names)) + + kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') + rsync_dest = 'root@%s:%s' % (location, kernel_dest) + self.status(msg='Copying kernel') + cliapp.runcmd( + ['rsync', kernel_src, rsync_dest]) + + # Link the kernel to the right place + self.status(msg='Creating links to kernel in tftp directory') + tftp_dir = os.path.join(self._nfsboot_root , 'tftp') + versioned_kernel_name = "%s-%s" % (hostname, version) + kernel_name = hostname + try: + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-f', kernel_dest, + os.path.join(tftp_dir, versioned_kernel_name)]) + + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-sf', versioned_kernel_name, + os.path.join(tftp_dir, kernel_name)]) + except cliapp.AppException: + raise cliapp.AppException('Could not create symlinks to the ' + 'kernel at %s in %s on %s' + % (kernel_dest, tftp_dir, location)) + + def copy_rootfs(self, temp_root, location, versioned_root, hostname): + rootfs_src = temp_root + '/' + orig_path = os.path.join(versioned_root, 'orig') + run_path = os.path.join(versioned_root, 'run') + + self.status(msg='Creating destination directories') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except cliapp.AppException: + raise cliapp.AppException('Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) + + self.status(msg='Creating \'orig\' rootfs') + cliapp.runcmd( + ['rsync', '-aXSPH', '--delete', rootfs_src, + 'root@%s:%s' % (location, orig_path)]) + + self.status(msg='Creating \'run\' rootfs') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + cliapp.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + cliapp.ssh_runcmd('root@%s' % location, + ['rm', '-rf', os.path.join(run_path, 'etc')]) + cliapp.ssh_runcmd('root@%s' % location, + ['cp', '-a', os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except cliapp.AppException: + raise cliapp.AppException('Could not create \'run\' rootfs' + ' from \'orig\'') + + self.status(msg='Linking \'default-run\' to latest system') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-sfn', run_path, + os.path.join(self._nfsboot_root, hostname, 'systems', + 'default-run')]) + except cliapp.AppException: + raise cliapp.AppException('Could not link \'default-run\' to %s' + % run_path) + + def configure_nfs(self, location, hostname): + exported_path = os.path.join(self._nfsboot_root, hostname) + exports_path = '/etc/exports' + # If that path is not already exported: + try: + cliapp.ssh_runcmd( + 'root@%s' % location, ['grep', '-q', exported_path, + exports_path]) + except cliapp.AppException: + ip_mask = '*' + options = 'rw,no_subtree_check,no_root_squash,async' + exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) + exports_append_sh = '''\ +set -eu +target="$1" +temp=$(mktemp) +cat "$target" > "$temp" +cat >> "$temp" +mv "$temp" "$target" +''' + cliapp.ssh_runcmd( + 'root@%s' % location, + ['sh', '-c', exports_append_sh, '--', exports_path], + feed_stdin=exports_string) + cliapp.ssh_runcmd( + 'root@%s' % location, ['systemctl', 'restart', + 'nfs-server.service']) + + def test_good_server(self, server): + # Can be ssh'ed into + try: + cliapp.ssh_runcmd('root@%s' % server, ['true']) + except cliapp.AppException: + raise cliapp.AppException('You are unable to ssh into server %s' + % server) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % server) + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['systemctl', 'is-enabled', + 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % server) + + # TFTP server exports /srv/nfsboot/tftp + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export ' + '/srv/nfsboot/tftp' % server) + +NFSBootWriteExtension().run() + diff --git a/rawdisk.write b/rawdisk.write new file mode 100755 index 0000000..a43a9cc --- /dev/null +++ b/rawdisk.write @@ -0,0 +1,95 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for raw disk images.''' + + +import cliapp +import os +import sys +import time +import tempfile + +import morphlib.writeexts + + +class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a raw disk image during Morph's deployment. + + If the image already exists, it is upgraded. + + The location command line argument is the pathname of the disk image + to be created/upgraded. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + if os.path.isfile(location): + self.upgrade_local_system(location, temp_root) + else: + self.create_local_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % location) + + def upgrade_local_system(self, raw_disk, temp_root): + mp = self.mount(raw_disk) + + version_label = self.get_version_label(mp) + self.status(msg='Updating image to a new version with label %s' % + version_label) + + version_root = os.path.join(mp, 'systems', version_label) + os.mkdir(version_root) + + old_orig = os.path.join(mp, 'systems', 'factory', 'orig') + new_orig = os.path.join(version_root, 'orig') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + + cliapp.runcmd( + ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', + temp_root + os.path.sep, new_orig]) + + self.create_run(version_root) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_extlinux(mp, version_label) + + self.unmount(mp) + + def get_version_label(self, mp): + version_label = os.environ.get('VERSION_LABEL') + + if version_label is None: + self.unmount(mp) + raise cliapp.AppException('VERSION_LABEL was not given') + + if os.path.exists(os.path.join(mp, 'systems', version_label)): + self.unmount(mp) + raise cliapp.AppException('VERSION_LABEL %s already exists' + % version_label) + + return version_label + + +RawDiskWriteExtension().run() + diff --git a/set-hostname.configure b/set-hostname.configure new file mode 100755 index 0000000..e44c5d5 --- /dev/null +++ b/set-hostname.configure @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Set hostname on system from HOSTNAME. + + +set -e + +if [ -n "$HOSTNAME" ] +then + echo "$HOSTNAME" > "$1/etc/hostname" +fi + diff --git a/simple-network.configure b/simple-network.configure new file mode 100755 index 0000000..b98b202 --- /dev/null +++ b/simple-network.configure @@ -0,0 +1,143 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''A Morph deployment configuration extension to handle /etc/network/interfaces + +This extension prepares /etc/network/interfaces with the interfaces specified +during deployment. + +If no network configuration is provided, eth0 will be configured for DHCP +with the hostname of the system. +''' + + +import os +import sys +import cliapp + +import morphlib + + +class SimpleNetworkError(morphlib.Error): + '''Errors associated with simple network setup''' + pass + + +class SimpleNetworkConfigurationExtension(cliapp.Application): + '''Configure /etc/network/interfaces + + Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces. + ''' + + def process_args(self, args): + network_config = os.environ.get( + "NETWORK_CONFIG", "lo:loopback;eth0:dhcp,hostname=$(hostname)") + + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + + stanzas = self.parse_network_stanzas(network_config) + iface_file = self.generate_iface_file(stanzas) + + with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: + f.write(iface_file) + + def generate_iface_file(self, stanzas): + """Generate an interfaces file from the provided stanzas. + + The interfaces will be sorted by name, with loopback sorted first. + """ + + def cmp_iface_names(a, b): + a = a['name'] + b = b['name'] + if a == "lo": + return -1 + elif b == "lo": + return 1 + else: + return cmp(a,b) + + return "\n".join(self.generate_iface_stanza(stanza) + for stanza in sorted(stanzas, cmp=cmp_iface_names)) + + def generate_iface_stanza(self, stanza): + """Generate an interfaces stanza from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + lines = ["auto %s" % name, "iface %s inet %s" % (name, itype)] + lines += [" %s %s" % elem for elem in stanza['args'].items()] + lines += [""] + return "\n".join(lines) + + + def parse_network_stanzas(self, config): + """Parse a network config environment variable into stanzas. + + Network config stanzas are semi-colon separated. + """ + + return [self.parse_network_stanza(s) for s in config.split(";")] + + def parse_network_stanza(self, stanza): + """Parse a network config stanza into name, type and arguments. + + Each stanza is of the form name:type[,arg=value]... + + For example: + lo:loopback + eth0:dhcp + eth1:static,address=10.0.0.1,netmask=255.255.0.0 + """ + elements = stanza.split(",") + lead = elements.pop(0).split(":") + if len(lead) != 2: + raise SimpleNetworkError("Stanza '%s' is missing its type" % + stanza) + iface = lead[0] + iface_type = lead[1] + + if iface_type not in ['loopback', 'static', 'dhcp']: + raise SimpleNetworkError("Stanza '%s' has unknown interface type" + " '%s'" % (stanza, iface_type)) + + argpairs = [element.split("=", 1) for element in elements] + output_stanza = { "name": iface, + "type": iface_type, + "args": {} } + for argpair in argpairs: + if len(argpair) != 2: + raise SimpleNetworkError("Stanza '%s' has bad argument '%r'" + % (stanza, argpair.pop(0))) + if argpair[0] in output_stanza["args"]: + raise SimpleNetworkError("Stanza '%s' has repeated argument" + " %s" % (stanza, argpair[0])) + output_stanza["args"][argpair[0]] = argpair[1] + + return output_stanza + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + +SimpleNetworkConfigurationExtension().run() diff --git a/ssh-rsync.write b/ssh-rsync.write new file mode 100755 index 0000000..6fe1153 --- /dev/null +++ b/ssh-rsync.write @@ -0,0 +1,181 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for upgrading systems over ssh.''' + + +import cliapp +import os +import sys +import time +import tempfile + +import morphlib.writeexts + +class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): + + '''Upgrade a running baserock system with ssh and rsync. + + It assumes the system is baserock-based and has a btrfs partition. + + The location command line argument is the 'user@hostname' string + that will be passed to ssh and rsync + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.check_valid_target(location) + self.upgrade_remote_system(location, temp_root) + + def upgrade_remote_system(self, location, temp_root): + root_disk = self.find_root_disk(location) + version_label = os.environ.get('VERSION_LABEL') + + try: + self.status(msg='Creating remote mount point') + remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + + self.status(msg='Mounting root disk') + cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) + + version_root = os.path.join(remote_mnt, 'systems', version_label) + run_dir = os.path.join(version_root, 'run') + orig_dir = os.path.join(version_root, 'orig') + try: + self.status(msg='Creating %s' % version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + + self.create_remote_orig(location, version_root, remote_mnt, + temp_root) + + self.status(msg='Creating "run" subvolume') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', + 'snapshot', orig_dir, run_dir]) + + self.install_remote_kernel(location, version_root, temp_root) + except Exception as e: + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', run_dir]) + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', orig_dir]) + cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) + except: + pass + raise e + + if self.bootloader_is_wanted(): + self.update_remote_extlinux(location, remote_mnt, + version_label) + except: + raise + else: + self.status(msg='Removing temporary mounts') + cliapp.ssh_runcmd(location, ['umount', root_disk]) + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + + def update_remote_extlinux(self, location, remote_mnt, version_label): + '''Install/reconfigure extlinux on location''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(remote_mnt, 'extlinux.conf') + temp_file = tempfile.mkstemp()[1] + with open(temp_file, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('append root=/dev/sda ' + 'rootflags=subvol=systems/' + version_label + '/run ' + 'init=/sbin/init rw\n') + + cliapp.ssh_runcmd(location, ['mv', config, config+'~']) + + try: + cliapp.runcmd(['rsync', '-a', temp_file, + '%s:%s' % (location, config)]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, ['mv', config+'~', config]) + except: + pass + raise e + + def create_remote_orig(self, location, version_root, remote_mnt, + temp_root): + '''Create the subvolume version_root/orig on location''' + + self.status(msg='Creating "orig" subvolume') + old_orig = self.get_old_orig(location, remote_mnt) + new_orig = os.path.join(version_root, 'orig') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) + + cliapp.runcmd(['rsync', '-a', '--checksum', '--numeric-ids', + '--delete', temp_root, '%s:%s' % (location, new_orig)]) + + def get_old_orig(self, location, remote_mnt): + '''Identify which subvolume to snapshot from''' + + # rawdisk upgrades use 'factory' + return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + + def find_root_disk(self, location): + '''Read /proc/mounts on location to find which device contains "/"''' + + self.status(msg='Finding device that contains "/"') + contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + for line in contents.splitlines(): + line_words = line.split() + if (line_words[1] == '/' and line_words[0] != 'rootfs'): + return line_words[0] + + def install_remote_kernel(self, location, version_root, temp_root): + '''Install the kernel in temp_root inside version_root on location''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'kernel') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['rsync', '-a', try_path, + '%s:%s' % (location, kernel_dest)]) + + def check_valid_target(self, location): + try: + cliapp.ssh_runcmd(location, ['true']) + except Exception as e: + raise cliapp.AppException('%s does not respond to ssh:\n%s' + % (location, e)) + + try: + cliapp.ssh_runcmd(location, ['test', '-d', '/baserock']) + except: + raise cliapp.AppException('%s is not a baserock system' % location) + + try: + cliapp.ssh_runcmd(location, ['which', 'rsync']) + except: + raise cliapp.AppException('%s does not have rsync') + +SshRsyncWriteExtension().run() diff --git a/ssh.configure b/ssh.configure new file mode 100755 index 0000000..2f3167e --- /dev/null +++ b/ssh.configure @@ -0,0 +1,162 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''A Morph deployment configuration to copy SSH keys. + +Keys are copied from the host to the new system. +''' + +import cliapp +import os +import sys +import shutil +import glob +import re +import logging + +import morphlib + +class SshConfigurationExtension(cliapp.Application): + + '''Copy over SSH keys to new system from host. + + The extension requires SSH_KEY_DIR to be set at the command line as it + will otherwise pass with only a status update. SSH_KEY_DIR should be + set to the location of the SSH keys to be passed to the new system. + + ''' + + def process_args(self, args): + if 'SSH_KEY_DIR' in os.environ: + # Copies ssh_host keys. + key = 'ssh_host_*_key' + mode = 0755 + dest = os.path.join(args[0], 'etc/ssh/') + sshhost, sshhostpub = self.find_keys(key) + if sshhost or sshhostpub: + self.check_dir(dest, mode) + self.copy_keys(sshhost, sshhostpub, dest) + + # Copies root keys. + key = 'root_*_key' + mode = 0700 + dest = os.path.join(args[0], 'root/.ssh/') + roothost, roothostpub = self.find_keys(key) + key = 'root_authorized_key_*.pub' + authkey, bleh = self.find_keys(key) + if roothost or roothostpub: + self.check_dir(dest, mode) + self.copy_rename_keys(roothost, + roothostpub, dest, 'id_', [5, 4]) + if authkey: + self.check_dir(dest, mode) + self.comb_auth_key(authkey, dest) + + # Fills the known_hosts file + key = 'root_known_host_*_key.pub' + src = os.path.join(os.environ['SSH_KEY_DIR'], key) + known_hosts_keys = glob.glob(src) + if known_hosts_keys: + self.check_dir(dest, mode) + known_hosts_path = os.path.join(dest, 'known_hosts') + with open(known_hosts_path, "a") as known_hosts_file: + for filename in known_hosts_keys: + hostname = re.search('root_known_host_(.+?)_key.pub', + filename).group(1) + known_hosts_file.write(hostname + " ") + with open(filename, "r") as f: + shutil.copyfileobj(f, known_hosts_file) + + else: + self.status(msg="No SSH key directory found.") + pass + + def find_keys(self, key_name): + '''Uses glob to find public and + private SSH keys and returns their path''' + + src = os.path.join(os.environ['SSH_KEY_DIR'], key_name) + keys = glob.glob(src) + pubkeys = glob.glob(src + '.pub') + if not (keys or pubkeys): + self.status(msg="No SSH keys of pattern %(src)s found.", src=src) + return keys, pubkeys + + def check_dir(self, dest, mode): + '''Checks if destination directory exists + and creates it if necessary''' + + if os.path.exists(dest) == False: + self.status(msg="Creating SSH key directory: %(dest)s", dest=dest) + os.mkdir(dest) + os.chmod(dest, mode) + else: + pass + + def copy_keys(self, keys, pubkeys, dest): + '''Copies SSH keys to new VM''' + + for key in keys: + shutil.copy(key, dest) + path = os.path.join(dest, os.path.basename(key)) + os.chmod(path, 0600) + for key in pubkeys: + shutil.copy(key, dest) + path = os.path.join(dest, os.path.basename(key)) + os.chmod(path, 0644) + + def copy_rename_keys(self, keys, pubkeys, dest, new, snip): + '''Copies SSH keys to new VM and renames them''' + + st, fi = snip + for key in keys: + base = os.path.basename(key) + s = len(base) + nw_dst = os.path.join(dest, new + base[st:s-fi]) + shutil.copy(key, nw_dst) + os.chmod(nw_dst, 0600) + for key in pubkeys: + base = os.path.basename(key) + s = len(base) + nw_dst = os.path.join(dest, new + base[st:s-fi-4]) + shutil.copy(key, nw_dst + '.pub') + os.chmod(nw_dst + '.pub', 0644) + + def comb_auth_key(self, keys, dest): + '''Combines authorized_keys file in new VM''' + + dest = os.path.join(dest, 'authorized_keys') + fout = open(dest, 'a') + for key in keys: + fin = open(key, 'r') + data = fin.read() + fout.write(data) + fin.close() + fout.close() + os.chmod(dest, 0600) + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + +SshConfigurationExtension().run() diff --git a/tar.write b/tar.write new file mode 100755 index 0000000..7a2f01e --- /dev/null +++ b/tar.write @@ -0,0 +1,21 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# A Morph write extension to deploy to a .tar file + +set -eu + +tar -C "$1" -cf "$2" diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write new file mode 100755 index 0000000..cb17b69 --- /dev/null +++ b/virtualbox-ssh.write @@ -0,0 +1,148 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to VirtualBox via ssh. + +VirtualBox is assumed to be running on a remote machine, which is +accessed over ssh. The machine gets created, but not started. + +''' + + +import cliapp +import os +import re +import sys +import time +import tempfile +import urlparse + +import morphlib.writeexts + + +class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a VirtualBox virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + vbox+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which VirtualBox is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run VirtualBox's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vdi_path = self.parse_location(location) + autostart = self.parse_autostart() + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + + try: + self.transfer_and_convert_to_vdi( + raw_disk, ssh_host, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, + autostart) + except BaseException: + sys.stderr.write('Error deploying to VirtualBox') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'vbox+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): + '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' + + self.status(msg='Transfer disk and convert to VDI') + with open(raw_disk, 'rb') as f: + cliapp.runcmd( + ['ssh', ssh_host, + 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, + str(os.path.getsize(raw_disk))], + stdin=f) + + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): + '''Create the VirtualBox virtual machine.''' + + self.status(msg='Create VirtualBox virtual machine') + + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + + commands = [ + ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', + '--register'], + ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, + '--nic1', 'nat'], + ['storagectl', vm_name, '--name', '"SATA Controller"', + '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], + ['storageattach', vm_name, '--storagectl', '"SATA Controller"', + '--port', '0', '--device', '0', '--type', 'hdd', '--medium', + vdi_path], + ] + + attach_disks = self.parse_attach_disks() + for device_no, disk in enumerate(attach_disks, 1): + cmd = ['storageattach', vm_name, + '--storagectl', '"SATA Controller"', + '--port', str(device_no), + '--device', '0', + '--type', 'hdd', + '--medium', disk] + commands.append(cmd) + + if autostart: + commands.append(['startvm', vm_name]) + + for command in commands: + argv = ['ssh', ssh_host, 'VBoxManage'] + command + cliapp.runcmd(argv) + + +VirtualBoxPlusSshWriteExtension().run() + |