diff options
Diffstat (limited to 'morphlib/plugins/deploy_plugin.py')
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 310 |
1 files changed, 150 insertions, 160 deletions
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 63cc4688..ea84d9ec 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -21,11 +21,94 @@ import sys import tarfile import tempfile import uuid +import warnings import cliapp import morphlib +def configuration_for_system(system_id, vars_from_commandline, + deploy_defaults, deploy_params): + '''Collect all configuration variables for deploying one system. + + This function collects variables from the following places: + + - the values specified in the 'deploy-defaults' section of the cluster + .morph file. + - values specified in the stanza for the system in the cluster.morph file + - environment variables of the running `morph deploy` process + - values specified on the `morph deploy` commandline, for example + `mysystem.HOSTNAME=foo`. + + Later values override earlier ones, so 'deploy-defaults' has the lowest + precidence and the `morph deploy` commandline has highest precidence. + + ''' + commandline_vars_for_system = [ + pair[len(system_id)+1:] for pair in vars_from_commandline + if pair.startswith(system_id)] + + user_env = morphlib.util.parse_environment_pairs( + os.environ, commandline_vars_for_system) + + # Order is important here: the second dict overrides the first, the third + # overrides the second. + final_env = dict(deploy_defaults.items() + + deploy_params.items() + + user_env.items()) + + morphlib.util.sanitize_environment(final_env) + + return final_env + + +def deployment_type_and_location(system_id, config, is_upgrade): + '''Get method and location for deploying a given system. + + The rules for this depend on whether the user is running `morph deploy` + (initial deployment) or `morph upgrade`. The latter honours 'upgrade-type' + and 'upgrade-location' if they are set, falling back to 'type' and + 'location' if they are not. The former only honours 'type' and 'location'. + + In the past, only the 'type' and 'location' fields existed. So `morph + upgrade` needs to handle the case where only these are set, to avoid + breaking existing cluster .morph files. + + ''' + if is_upgrade and ('upgrade-type' in config or 'upgrade-location' in \ + config): + if 'upgrade-type' not in config: + raise morphlib.Error( + '"upgrade-location" was set for system %s, but not ' + '"upgrade-type"' % system_id) + + if 'upgrade-location' not in config: + raise morphlib.Error( + '"upgrade-type" was set for system %s, but not ' + '"upgrade-location"' % system_id) + + deployment_type = config['upgrade-type'] + location = config['upgrade-location'] + else: + if 'type' not in config: + raise morphlib.Error( + '"type" is undefined for system "%s"' % system_id) + + if 'location' not in config: + raise morphlib.Error( + '"location" is undefined for system "%s"' % system_id) + + if is_upgrade: + warnings.warn( + '"upgrade-type" and "upgrade-location" were not specified for ' + 'system %s, using "type" and "location"\n' % system_id) + + deployment_type = config['type'] + location = config['location'] + + return deployment_type, location + + class DeployPlugin(cliapp.Plugin): def enable(self): @@ -87,11 +170,14 @@ class DeployPlugin(cliapp.Plugin): system and each system has at least the following keys: * **type**: identifies the type of development e.g. (kvm, - nfsboot) (see below). + pxeboot) (see below). * **location**: where the deployed system should end up at. The syntax depends on the deployment type (see below). - Any additional item on the dictionary will be added to the - environment as `KEY=VALUE`. + + Optionally, it can specify **upgrade-type** and + **upgrade-location** as well for use with `morph upgrade`. Any + additional item on the dictionary will be added to the environment + as `KEY=VALUE`. * **deploy-defaults**: allows multiple deployments of the same system to share some settings, when they can. Default settings @@ -107,14 +193,16 @@ class DeployPlugin(cliapp.Plugin): cluster-foo-x86_64-1: type: kvm location: kvm+ssh://user@host/x86_64-1/x86_64-1.img + upgrade-type: ssh-rsync + upgrade-location: root@localhost HOSTNAME: cluster-foo-x86_64-1 DISK_SIZE: 4G RAM_SIZE: 4G VCPUS: 2 - morph: devel-system-armv7-highbank deploy-defaults: - type: nfsboot - location: cluster-foo-nfsboot-server + type: pxeboot + location: cluster-foo-pxeboot-server deploy: cluster-foo-armv7-1: HOSTNAME: cluster-foo-armv7-1 @@ -122,91 +210,24 @@ class DeployPlugin(cliapp.Plugin): HOSTNAME: cluster-foo-armv7-2 Each system defined in a cluster morphology can be deployed in - multiple ways (`type` in a cluster morphology). Morph provides - the following types of deployment: - - * `tar` where Morph builds a tar archive of the root file system. - - * `rawdisk` where Morph builds a raw disk image and sets up the - image with a bootloader and configuration so that it can be - booted. Disk size is set with `DISK_SIZE` (see below). - - * `virtualbox-ssh` where Morph creates a VirtualBox disk image, - and creates a new virtual machine on a remote host, accessed - over ssh. Disk and RAM size are set with `DISK_SIZE` and - `RAM_SIZE` (see below). + multiple ways (`type` in a cluster morphology). These methods are + implemented by .write extensions. There are some built into Morph, + and you can also store them in a definitions.git repo. - * `kvm`, which is similar to `virtualbox-ssh`, but uses libvirt - and KVM instead of VirtualBox. Disk and RAM size are set with - `DISK_SIZE` and `RAM_SIZE` (see below). - - * `nfsboot` where Morph creates a system to be booted over - a network. - - * `ssh-rsync` where Morph copies a binary delta over to the target - system and arranges for it to be bootable. This requires - `system-version-manager` from the tbdiff chunk - - * `initramfs`, where Morph turns the system into an initramfs image, - suitable for being used as the early userland environment for a - system to be able to locate more complicated storage for its root - file-system, or on its own for diskless deployments. - - There are additional extensions that currently live in the Baserock - definitions repo (baserock:baserock/definitions). These include: - - * `image-package` where Morph creates a tarball that includes scripts - that can be used to make disk images outside of a Baserock - environment. The example in definitions.git will create scripts for - generating disk images and installing to existing disks. - - * `sdk` where Morph generates something resembing a BitBake SDK, which - provides a toolchain for building software to target a system built - by Baserock, from outside of a Baserock environment. This creates a - self-extracting shell archive which you pass a directory to extract - to, and inside that has a shell snippet called - environment-setup-$TARGET which can be used to set environment - variables to use the toolchain. - - * `pxeboot` where Morph temporarily network-boots the system you are - deploying, so it can install a more permanent system onto local - storage. + See `morph help-extensions` for a full list of these extensions. If you + run this command in a system branch, it will list those that are + available in the definitions.git repo that is checked out as well as + those built-in to Morph. Each extension can provide its own + documentation. To see help for the 'tar' write extension, for example, + run `morph help tar.write`. In addition to the deployment type, the user must also give a value for `location`. Its syntax depends on the deployment - types. The deployment types provided by Morph use the - following syntaxes: - - * `tar`: pathname to the tar archive to be created; for - example, `/home/alice/testsystem.tar` + method. See the help file for the given write extension to find out + how to format the 'location' field. - * `rawdisk`: pathname to the disk image to be created; for - example, `/home/alice/testsystem.img` - - * `virtualbox-ssh` and `kvm`: a custom URL scheme that - provides the target host machine (the one that runs - VirtualBox or `kvm`), the name of the new virtual machine, - and the location on the target host of the virtual disk - file. The target host is accessed over ssh. For example, - `vbox+ssh://alice@192.168.122.1/testsys/home/alice/testsys.vdi` - or `kvm+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` - where - - * `alice@192.168.122.1` is the target as given to ssh, - **from within the development host** (which may be - different from the target host's normal address); - - * `testsys` is the new VM's name; - - * `/home/alice/testsys.vdi` and `/home/alice/testys.img` are - the pathnames of the disk image files on the target host. - - * `nfsboot`: the address of the nfsboot server. (Note this is just - the _address_ of the trove, _not_ `user@...`, since `root@` will - automatically be prepended to the server address.) - - In addition to the `location`parameter, deployments can take additional - `KEY=VALUE` parameters. These can be provided in the following ways: + Deployments take additional `KEY=VALUE` parameters as well. These can + be provided in the following ways: 1. In the cluster definition file, e.g. @@ -230,52 +251,8 @@ class DeployPlugin(cliapp.Plugin): -ve `no`, `0`, `false`; - The following `KEY=VALUE` parameters are supported for `rawdisk`, - `virtualbox-ssh` and `kvm` and deployment types: - - * `DISK_SIZE=X` to set the size of the disk image. `X` should use a - suffix of `K`, `M`, or `G` (in upper or lower case) to indicate - kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would - create a 100 gigabyte disk image. **This parameter is mandatory**. - - The `kvm` and `virtualbox-ssh` deployment types support an additional - parameter: - - * `RAM_SIZE=X` to set the size of virtual RAM for the virtual - machine. `X` is interpreted in the same was as `DISK_SIZE`, - and defaults to `1G`. - - * `AUTOSTART=<VALUE>` - allowed values are `yes` and `no` - (default) - - For the `nfsboot` write extension, - - * the following `KEY=VALUE` pairs are mandatory - - * `NFSBOOT_CONFIGURE=yes` (or any non-empty value). This - enables the `nfsboot` configuration extension (see - below) which MUST be used when using the `nfsboot` - write extension. - - * `HOSTNAME=<STRING>` a unique identifier for that system's - `nfs` root when it's deployed on the nfsboot server - the - extension creates a directory with that name for the `nfs` - root, and stores kernels by that name for the tftp server. - - * the following `KEY=VALUE` pairs are optional - - * `VERSION_LABEL=<STRING>` - set the name of the system - version being deployed, when upgrading. Defaults to - "factory". - - Each deployment type is implemented by a **write extension**. The - ones provided by Morph are listed above, but users may also - create their own by adding them in the same git repository - and branch as the system morphology. A write extension is a - script that does whatever is needed for the deployment. A write - extension is passed two command line parameters: the name of an - unpacked directory tree that contains the system files (after - configuration, see below), and the `location` parameter. + Some extensions require certain parameters to be set, be sure to read + the documentation of the extension you are using. Regardless of the type of deployment, the image may be configured for a specific deployment by using **configuration @@ -293,19 +270,22 @@ class DeployPlugin(cliapp.Plugin): Configuration extensions are scripts that get the unpacked directory tree of the system as their parameter, and do whatever - is needed to configure the tree. - - Morph provides the following configuration extension built in: - - * `set-hostname` sets the hostname of the system to the value - of the `HOSTNAME` variable. - * `nfsboot` configures the system for nfsbooting. This MUST - be used when deploying with the `nfsboot` write extension. + is needed to configure the tree. Available .configuration extensions + can be found with `morph help-extensions`, as with .write extensions. Any `KEY=VALUE` parameters given in `deploy` or `deploy-defaults` sections of the cluster morphology, or given through the command line are set as environment variables when either the configuration or the - write extension runs (except `type` and `location`). + write extension runs. + + You can write your own .write and .configure extensions, in any + format that Morph can execute at deploy-time. They must be committed + to your definitions.git repository for Morph to find them. + A .configure extension is passed one argument: the path to an unpacked + directory tree containing the system files. A write extension is passed + two command line parameters: the path to the unpacked system, and the + `location` parameter. The .configure and .write extensions have full + 'root' access to the build machine, so write them carefully! Deployment configuration is stored in the deployed system as /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET @@ -453,31 +433,16 @@ class DeployPlugin(cliapp.Plugin): system_status_prefix, system_id) self.app.status_prefix = deployment_status_prefix try: - user_env = morphlib.util.parse_environment_pairs( - os.environ, - [pair[len(system_id)+1:] - for pair in env_vars - if pair.startswith(system_id)]) - - final_env = dict(deploy_defaults.items() + - deploy_params.items() + - user_env.items()) + final_env = configuration_for_system( + system_id, env_vars, deploy_defaults, deploy_params) is_upgrade = ('yes' if self.app.settings['upgrade'] else 'no') final_env['UPGRADE'] = is_upgrade - deployment_type = final_env.pop('type', None) - if not deployment_type: - raise morphlib.Error('"type" is undefined ' - 'for system "%s"' % system_id) - - location = final_env.pop('location', None) - if not location: - raise morphlib.Error('"location" is undefined ' - 'for system "%s"' % system_id) + deployment_type, location = deployment_type_and_location( + system_id, final_env, self.app.settings['upgrade']) - morphlib.util.sanitize_environment(final_env) self.check_deploy(root_repo_dir, ref, deployment_type, location, final_env) system_tree = self.setup_deploy(build_command, @@ -508,7 +473,32 @@ class DeployPlugin(cliapp.Plugin): def upgrade(self, args): '''Upgrade an existing set of instances using built images. - See `morph help deploy` for documentation. + This command is very similar to `morph deploy`. Please read `morph help + deploy` first for an introduction to deployment (cluster) .morph files. + + To allow upgrading a system after the initial deployment, you need to + set two extra fields: **upgrade-type** and **upgrade-location**. + + The most common .write extension for upgrades is the `ssh-rsync` write + extension. See `morph help ssh-rsync.write` to read its documentation + + To deploy a development system that can then deploy upgrades to itself + using passwordless SSH access, adapt the following example cluster: + + name: devel-system + kind: cluster + systems: + - morph: systems/devel-system-x86_64-generic.morph + deploy: + devel-system: + type: kvm + location: kvm+ssh://... + + upgrade-type: ssh-rsync + upgrade-location: root@localhost + + HOSTNAME: my-devel-system + .. ''' |