Add mason 'build', 'test' and 'upload' as Zuul pluginsbaserock/mason-v2

Much of this code comes from the release-* scripts in
Baserock's defintions repo.

At the moment, only build, build_test and artifact_upload plugins
are implemented. Also note that the test build run on a deployed
instance in the build_test plugin currently builds master rather
than the ref of the proposed change being tested. This is still
adequate as that system is itself built from the ref of the
proposed change.

1 files changed, 188 insertions, 0 deletions

diff --git a/mason/tests/build_test.py b/mason/tests/build_test.py
new file mode 100644
index 0000000..cdf98c3
--- /dev/null
+++ b/mason/tests/build_test.py
@@ -0,0 +1,188 @@
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import json
+import logging
+import os
+import urlparse
+
+import mason
+
+
+class Runner(mason.runners.JobRunner):
+
+    """Run the Build test
+
+    This handles running the Build test, which is used to ensure
+    that Baserock can build Baserock.
+    """
+
+    log = logging.getLogger("mason.tests.build_test.Runner")
+
+    def __init__(self, worker_server, plugin_config, job_name):
+        super(Runner, self).__init__(worker_server, plugin_config, job_name)
+
+        self.total_steps = 4
+
+    def run_job(self):
+        self.log.debug('Got job: %s', self.job_arguments)
+
+        self.log.info('Step 1: Creating a workspace')
+        self._create_workspace()
+
+        self.log.info('Step 2: Create the log directory')
+        self._prepare_log_dir()
+
+        self.log.info('Step 3: Deploy and test the systems')
+        try:
+            self._deploy_and_test_systems()
+        except Exception as e:
+            self.log.info('Failed to deploy and test systems: Exception %s', e)
+            self._remove_workspace()
+            raise e
+
+        self.log.info('Step 4: Clean up')
+        self._clean_up()
+
+    @staticmethod
+    def _run_tests(instance, system_path, system_morph,
+                   (trove_host, trove_id), systems, logfile):
+        instance.logfile = logfile
+        has_cloudinit = any('cloudinit-support' in si['morph'] for si in system_morph['strata'])
+        instance.wait_until_online(has_cloudinit)
+
+        tests = []
+        def baserock_build_test(instance):
+            instance.runcmd(['git', 'config', '--global', 'user.name',
+                             'Test Instance of %s' % instance.deployment.name],
+                             stdin=None, stdout=logfile, stderr=logfile)
+            instance.runcmd(['git', 'config', '--global', 'user.email',
+                             'ci-test@%s' % instance.config['HOSTNAME']],
+                             stdin=None, stdout=logfile, stderr=logfile)
+            instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
+                             '/src/tmp'],
+                             stdin=None, stdout=logfile, stderr=logfile)
+            def morph_cmd(*args, **kwargs):
+                # TODO: decide whether to use cached artifacts or not by
+                # adding --artifact-cache-server= --cache-server=
+                argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
+                        '--tempdir=/src/tmp', '--log-max=100M',
+                        '--trove-host', trove_host, '--trove-id', trove_id,
+                        '--local-changes=ignore']
+                argv.extend(args)
+                instance.runcmd(argv, **kwargs)
+
+            repo = 'baserock:baserock/definitions'
+            ref = 'master'
+
+            morph_cmd('init', '/src/ws',
+                      stdin=None, stdout=logfile, stderr=logfile)
+            chdir = '/src/ws'
+
+            morph_cmd('checkout', repo, ref, chdir=chdir,
+                      stdin=None, stdout=logfile, stderr=logfile)
+            # TODO: Add a morph subcommand that gives the path to the root repository.
+            chdir = os.path.join(chdir, ref, 'baserock', 'baserock',
+                                 'definitions')
+
+            instance.runcmd(['git', 'reset', '--hard', 'HEAD'], chdir=chdir,
+                            stdin=None, stdout=logfile, stderr=logfile)
+            logfile.write('Building test systems for %s\n' % system_path)
+            for to_build_path, to_build_morph in systems.iteritems():
+                if to_build_morph['arch'] == system_morph['arch']:
+                    logfile.write('Test building %s\n' % to_build_path)
+                    logfile.flush()
+                    morph_cmd('build', to_build_path, chdir=chdir,
+                              stdin=None, stdout=logfile, stderr=logfile)
+            logfile.write('Finished Building test systems\n')
+            logfile.flush()
+
+        # TODO: Match the systems with a regex in config?
+        if 'devel' in system_path:
+            tests.append(baserock_build_test)
+
+        for test in tests:
+            test(instance)
+
+    @mason.util.job_step
+    def _prepare_log_dir(self):
+        self.logdir = '/var/www/logs/%s-%s/build_test' % \
+                (self.project, self.commit[:7])
+        if not os.path.exists(self.logdir):
+            os.makedirs(self.logdir)
+
+    @mason.util.job_step
+    def _deploy_and_test_systems(self):
+        config = self.plugin_config['config']
+        infrastructure = config['test-infrastructure-type']
+        build_test_config = (config['trove-host'],
+                             config['trove-id'])
+        cluster_path = config['cluster-morphology']
+        cluster = self.morph_helper.load_morphology(cluster_path)
+        systems = dict(self.morph_helper.load_cluster_systems(cluster))
+
+        deployment_hosts = {}
+        for host_config in config['deployment-host']:
+            arch, address = host_config.split(':', 1)
+            user, address = address.split('@', 1)
+            address, disk_path = address.split(':', 1)
+            if user == '':
+                user = 'root'
+            deployment_hosts[arch] = mason.util.VMHost(
+                user, address, disk_path)
+
+        for system_path, deployment_name, deployment_config in \
+            self.morph_helper.iterate_cluster_deployments(cluster):
+
+            system = systems[system_path]
+            # We can only test systems in KVM that have a BSP
+            if not any('bsp' in si['morph'] for si in system['strata']):
+                continue
+
+            # We can only test systems in KVM that we have a host for
+            if system['arch'] not in deployment_hosts:
+                continue
+
+            host = deployment_hosts[system['arch']]
+            log_path = os.path.join(self.logdir,
+                                    '%s-deploy.log' % system['name'])
+            net_id = config['openstack-network-id']
+            deployment = mason.deployment.Deployment(cluster_path,
+                                                     deployment_name,
+                                                     deployment_config, host,
+                                                     net_id, log_path)
+
+            os.chdir(self.defs_checkout)
+            try:
+                instance = deployment.deploy(infrastructure)
+            except Exception as e:
+                raise e
+            log_path = os.path.join(self.logdir,
+                                    '%s-test.log' % system['name'])
+            logfile = open(log_path, 'w+')
+            try:
+                self._run_tests(instance, system_path, system,
+                                build_test_config, systems, logfile)
+            except Exception as e:
+                logfile.write("Exception while running tests: %s\n" % (e))
+                raise e
+            finally:
+                instance.delete()
+                logfile.close()
+
+            # If we cancel, we don't want to have to wait for all
+            # the systems to be tested before we stop.
+            self._handle_cancellation()