From 404d5079b3d12db8224b54250ccf736ee5f5748d Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 9 Dec 2014 13:59:05 +0000 Subject: Fully implement test plugins --- mason/tests/artifact_upload.py | 152 ++++++++++++++++++++++------------------- mason/tests/build.py | 71 +++++++++++++++---- mason/tests/build_test.py | 118 ++++++++++++++++++++++++-------- 3 files changed, 226 insertions(+), 115 deletions(-) diff --git a/mason/tests/artifact_upload.py b/mason/tests/artifact_upload.py index 21d1093..aa6c56f 100644 --- a/mason/tests/artifact_upload.py +++ b/mason/tests/artifact_upload.py @@ -1,4 +1,17 @@ # Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import json @@ -6,76 +19,80 @@ import logging import os import urlparse -from turbo_hipster.lib import common -from turbo_hipster.lib import models +import mason - -#TODO: Less different instances of this would be nice -class MorphologyHelper(object): - - def __init__(self, path): - self.defs_repo = morphlib.gitdir.GitDirectory(path) - self.loader = morphlib.morphloader.MorphologyLoader() - self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) - - def load_morphology(self, path): - text = self.finder.read_morphology(path) - return self.loader.load_from_string(text) - - @classmethod - def iterate_systems(cls, systems_list): - for system in systems_list: - yield morphlib.util.sanitise_morphology_path(system['morph']) - if 'subsystems' in system: - for subsystem in cls.iterate_systems(system['subsystems']): - yield subsystem - - def iterate_cluster_deployments(cls, cluster_morph): - for system in cluster_morph['systems']: - path = morphlib.util.sanitise_morphology_path(system['morph']) - defaults = system.get('deploy-defaults', {}) - for name, options in system['deploy'].iteritems(): - config = dict(defaults) - config.update(options) - yield path, name, config - - def load_cluster_systems(self, cluster_morph): - for system_path in set(self.iterate_systems(cluster_morph['systems'])): - system_morph = self.load_morphology(system_path) - yield system_path, system_morph - - -#TODO: Deployment - - -class Runner(models.Task): +class Runner(mason.runners.JobRunner): """This thread handles running the build-deploy-build test, which is used to ensure that Baserock can build Baserock.""" - log = logging.getLogger("task_plugins.build_deploy_test.task.Runner") + log = logging.getLogger("mason.tests.artifact_upload.Runner") def __init__(self, worker_server, plugin_config, job_name): super(Runner, self).__init__(worker_server, plugin_config, job_name) - - self.total_steps = 5 - print self.job_arguments - - def do_job_steps(self): + self.config = self.plugin_config['config'] + self._set_defaults() + + self.total_steps = 2 + if self.config['upload-build-artifacts']: + self.total_steps += 1 + if self.config['upload-release-artifacts']: + self.total_steps += 1 + + def _set_defaults(self): + self.config['public-trove-host'] = \ + self.config.get('public-trove-host') or 'git.baserock.org' + self.config['public-trove-username'] = \ + self.config.get('public-trove-username') or 'root' + self.config['public-trove-artifact-dir'] = \ + self.config.get('public-trove-artifact-dir') \ + or '/home/cache/artifacts' + + self.config['download-server-address'] = \ + self.config.get('download-server-address') \ + or 'download.baserock.org' + self.config['download-server-username'] = \ + self.config.get('download-server-username') or 'root' + self.config['download-server-private-dir'] = \ + self.config.get('download-server-private-dir') \ + or '/srv/download.baserock.org/baserock/.publish-temp' + self.config['download-server-public-dir'] = \ + self.config.get('download-server-public-dir') \ + or '/srv/download.baserock.org/baserock' + + self.config['release-artifact-dir'] = \ + self.config.get('release-artifact-dir') or '.' + self.config['local-build-artifacts-dir'] = \ + self.config.get('local-build-artifacts-dir') or 'build-artifacts' + self.config['architecture'] = \ + self.config.get('architecture') or [] + + if 'upload-release-artifacts' not in self.config: + self.config['upload-release-artifacts'] = True + if 'upload-build-artifacts' not in self.config: + self.config['upload-build-artifacts'] = True + + def run_job(self): self.log.info('Step 1: Creating a workspace') self._create_workspace() - self.log.info('Step 2: Deploy and test the systems') - self._deploy_and_test_systems() + if self.config['upload-build-artifacts']: + self.log.info('Step 2: Publish the build artifacts') + self._publish_build_artifacts() - self.log.info('Step 3: Clean up') + if self.config['upload-release-artifacts']: + self.log.info('Step %d: Publish the release artifacts' % + (self.current_step + 1)) + self._publish_release_artifacts() + + self.log.info('Step %d: Clean up' % (self.current_step + 1)) self._clean_up() def _do_git_config(self): cliapp.runcmd(['git', 'config', 'user.name', 'Mason Test Runner']) cliapp.runcmd(['git', 'config', 'user.email', 'mason@test.runner']) - @common.task_step + @mason.util.job_step def _create_workspace(self): self.commit = self.job_arguments['ZUUL_COMMIT'] self.project = self.job_arguments['ZUUL_PROJECT'] @@ -89,6 +106,7 @@ class Runner(models.Task): url.hostname, '8080', self.project) + self.morph_helper = mason.util.MorphologyHelper(self.defs_checkout) self._do_git_config() cliapp.runcmd(['morph', 'init', self.workspace]) @@ -96,23 +114,17 @@ class Runner(models.Task): repo = 'http://%s:8080/%s' % (url.hostname, self.project) cliapp.runcmd(['morph', 'checkout', repo, self.commit], cwd=self.workspace) - @common.task_step - def _test_systems(self): - infrastructure = \ - self.plugin_config['config']['test-infrastructure-type'] - cmd = ['scripts/release-test'] - args = ['--deployment-host', - self.plugin_config['config']['deployment-host'], - '--trove-host', self.plugin_config['config']['trove-host'], - '--trove-id', self.plugin_config['config']['trove-id'], - '--test-ref', self.commit - ] - if infrastructure == 'openstack': - cmd = ['/usr/lib/mason/mason-test-os'] - args += ['--net-id', - self.plugin_config['config']['openstack-network-id']] - args += [self.plugin_config['config']['cluster-morphology']] - - @common.task_step + @mason.util.job_step + def _publish_build_artifacts(self): + publisher = mason.publishers.BuildArtifactPublisher( + self.config, self.defs_repo) + publisher.publish_build_artifacts() + + @mason.util.job_step + def _publish_release_artifacts(self): + publisher = mason.publishers.ReleaseArtifactPublisher(self.config) + publisher.publish_release_artifacts() + + @mason.util.job_step def _clean_up(self): cliapp.runcmd(['rm', '-rf', self.workspace]) diff --git a/mason/tests/build.py b/mason/tests/build.py index d347eb9..de5a178 100644 --- a/mason/tests/build.py +++ b/mason/tests/build.py @@ -1,10 +1,25 @@ # Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import json import logging import morphlib import os +import subprocess +import time import urlparse import mason @@ -14,20 +29,26 @@ class Build(object): """A single build instance.""" - def __init__(self, name, controller): + def __init__(self, name, controller, logfile): self.system_name = name self.controller = controller + self.log_path = logfile + self.logfile = open(logfile, 'w+') + #TODO: use distbuild not local build self.command = [ 'morph', 'build', self.system_name] def start(self): - self.process = subprocess.Popen(self.command) + self.process = subprocess.Popen(self.command, stdout=self.logfile, stderr=self.logfile) def completed(self): return (self.process.poll() is not None) + def close_log(self): + self.logfile.close() -class Runner(mason.util.JobRunner): + +class Runner(mason.runners.JobRunner): """This thread handles running the build-deploy-build test, which is used to ensure that Baserock can build Baserock.""" @@ -37,24 +58,27 @@ class Runner(mason.util.JobRunner): def __init__(self, worker_server, plugin_config, job_name): super(Runner, self).__init__(worker_server, plugin_config, job_name) - self.total_steps = 3 + self.total_steps = 4 - def do_job_steps(self): + def run_job(self): self.log.info('Step 1: Creating a workspace') self._create_workspace() - self.log.info('Step 2: Building the systems') + self.log.info('Step 2: Prepare build log directory') + self._prepare_build_log_dir() + + self.log.info('Step 3: Building the systems') self._build_systems() - #TODO: provide logs - self.log.info('Step 3: Clean up') + self.log.info('Step 4: Clean up') self._clean_up() - def _do_git_config(self): - cliapp.runcmd(['git', 'config', 'user.name', 'Mason Test Runner']) - cliapp.runcmd(['git', 'config', 'user.email', 'mason@test.runner']) + def _do_git_config(self, name='Mason Test Runner', email='mason@runner'): + cliapp.runcmd(['git', 'config', '--global', 'user.name', name]) + cliapp.runcmd(['git', 'config', '--global', 'user.email', email]) def _parse_controllers(self, conf): + print 'parsing controllers' controllers = {} for arch, addr in (item.split(':') for item in conf['controllers']): controllers[arch] = addr @@ -67,10 +91,14 @@ class Runner(mason.util.JobRunner): builds = [] for system_name in systems: system = self.morph_helper.load_morphology(system_name) + print 'loaded %s' % system_name if system['arch'] in controllers: - builds.append(Build(system_name, controllers['arch'])) + logfile = os.path.join(self.logdir, '%s.log' % system['name']) + builds.append(Build(system_name, controllers[system['arch']], logfile)) + print 'prepared builds' return builds + @mason.util.job_step def _create_workspace(self): self.commit = self.job_arguments['ZUUL_COMMIT'] self.project = self.job_arguments['ZUUL_PROJECT'] @@ -84,7 +112,6 @@ class Runner(mason.util.JobRunner): url.hostname, '8080', self.project) - self.morph_helper = mason.util.MorphologyHelper(self.defs_checkout) self._do_git_config() cliapp.runcmd(['morph', 'init', self.workspace]) @@ -92,7 +119,16 @@ class Runner(mason.util.JobRunner): repo = 'http://%s:8080/%s' % (url.hostname, self.project) cliapp.runcmd(['morph', 'checkout', repo, self.commit], cwd=self.workspace) + self.morph_helper = mason.util.MorphologyHelper(self.defs_checkout) + + @mason.util.job_step + def _prepare_build_log_dir(self): + self.logdir = '/var/www/logs/%s-%s/build' % \ + (self.project, self.commit[:7]) + if not os.path.exists(self.logdir): + os.makedirs(self.logdir) + @mason.util.job_step def _build_systems(self): builds = self._prepare_builds(self.plugin_config['config']) os.chdir(self.defs_checkout) @@ -104,12 +140,17 @@ class Runner(mason.util.JobRunner): fail = False for build in builds: + build.close_log() if build.process.returncode != 0: fail = True - sys.stderr.write( - 'Building failed for %s\n' % build.system_name) + logging.error('Building failed for %s. Log is at %s.' % + (build.system_name, build.log_path)) if fail: raise cliapp.AppException('Building of systems failed.') + @mason.util.job_step def _clean_up(self): + os.chdir('/root') + #TODO: don't do this in production + self._do_git_config(name='Adam Coldrick', email='adam.coldrick@codethink.co.uk') cliapp.runcmd(['rm', '-rf', self.workspace]) diff --git a/mason/tests/build_test.py b/mason/tests/build_test.py index 11cc30e..09ce7db 100644 --- a/mason/tests/build_test.py +++ b/mason/tests/build_test.py @@ -1,4 +1,17 @@ # Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import json @@ -9,7 +22,7 @@ import urlparse import mason -class Runner(mason.util.JobRunner): +class Runner(mason.runners.JobRunner): """This thread handles running the build test, which is used to ensure that Baserock can build Baserock.""" @@ -19,36 +32,43 @@ class Runner(mason.util.JobRunner): def __init__(self, worker_server, plugin_config, job_name): super(Runner, self).__init__(worker_server, plugin_config, job_name) - self.total_steps = 3 + self.total_steps = 4 - def do_job_steps(self): + def run_job(self): self.log.info('Step 1: Creating a workspace') self._create_workspace() - self.log.info('Step 2: Deploy and test the systems') + self.log.info('Step 2: Create the log directory') + self._prepare_log_dir() + + self.log.info('Step 3: Deploy and test the systems') self._deploy_and_test_systems() - self.log.info('Step 3: Clean up') + self.log.info('Step 4: Clean up') self._clean_up() - def _do_git_config(self): - cliapp.runcmd(['git', 'config', 'user.name', 'Mason Test Runner']) - cliapp.runcmd(['git', 'config', 'user.email', 'mason@test.runner']) + def _do_git_config(self, name='Mason Test Runner', email='mason@runner'): + cliapp.runcmd(['git', 'config', '--global', 'user.name', name]) + cliapp.runcmd(['git', 'config', '--global', 'user.email', email]) @staticmethod def _run_tests(instance, system_path, system_morph, (trove_host, trove_id, build_ref_prefix), - systems): + systems, logfile): + instance.log = logfile instance.wait_until_online() tests = [] def baserock_build_test(instance): instance.runcmd(['git', 'config', '--global', 'user.name', - 'Test Instance of %s' % instance.deployment.name]) + 'Test Instance of %s' % instance.deployment.name], + stdin=None, stdout=logfile, stderr=logfile) instance.runcmd(['git', 'config', '--global', 'user.email', - 'ci-test@%s' % instance.config['HOSTNAME']]) + 'ci-test@%s' % instance.config['HOSTNAME']], + stdin=None, stdout=logfile, stderr=logfile) instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', - '/src/tmp']) + '/src/tmp'], + stdin=None, stdout=logfile, stderr=logfile) def morph_cmd(*args, **kwargs): # TODO: decide whether to use cached artifacts or not by # adding --artifact-cache-server= --cache-server= @@ -62,24 +82,29 @@ class Runner(mason.util.JobRunner): repo = self.morph_helper.sb.root_repository_url ref = self.morph_helper.defs_repo.HEAD sha1 = self.morph_helper.defs_repo.resolve_ref_to_commit(ref) - morph_cmd('init', '/src/ws') + morph_cmd('init', '/src/ws', + stdin=None, stdout=logfile, stderr=logfile) chdir = '/src/ws' - morph_cmd('checkout', repo, ref, chdir=chdir) + morph_cmd('checkout', repo, ref, chdir=chdir, + stdin=None, stdout=logfile, stderr=logfile) # TODO: Add a morph subcommand that gives the path to the root repository. repo_path = os.path.relpath( self.morph_helper.sb.get_git_directory_name(repo), self.morph_helper.sb.root_directory) chdir = os.path.join(chdir, ref, repo_path) - instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) - print 'Building test systems for {sys}'.format(sys=system_path) + instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir, + stdin=None, stdout=logfile, stderr=logfile) + logfile.write('Building test systems for %s\n' % system_path) for to_build_path, to_build_morph in systems.iteritems(): if to_build_morph['arch'] == system_morph['arch']: - print 'Test building {path}'.format(path=to_build_path) + logfile.write('Test building %s\n' % to_build_path) + logfile.flush() morph_cmd('build', to_build_path, chdir=chdir, - stdin=None, stdout=None, stderr=None) - print 'Finished Building test systems' + stdin=None, stdout=logfile, stderr=logfile) + logfile.write('Finished Building test systems\n') + logfile.flush() # TODO: Match the systems with a regex in config? if 'devel' in system_path: @@ -88,6 +113,7 @@ class Runner(mason.util.JobRunner): for test in tests: test(instance) + @mason.util.job_step def _create_workspace(self): self.commit = self.job_arguments['ZUUL_COMMIT'] self.project = self.job_arguments['ZUUL_PROJECT'] @@ -101,8 +127,6 @@ class Runner(mason.util.JobRunner): url.hostname, '8080', self.project) - self.morph_helper = baserock_tests.MorphologyHelper( - self.defs_checkout) self._do_git_config() cliapp.runcmd(['morph', 'init', self.workspace]) @@ -110,16 +134,35 @@ class Runner(mason.util.JobRunner): repo = 'http://%s:8080/%s' % (url.hostname, self.project) cliapp.runcmd(['morph', 'checkout', repo, self.commit], cwd=self.workspace) + self.morph_helper = mason.util.MorphologyHelper(self.defs_checkout) + + @mason.util.job_step + def _prepare_log_dir(self): + self.logdir = '/var/www/logs/%s-%s/build_test' % \ + (self.project, self.commit[:7]) + if not os.path.exists(self.logdir): + os.makedirs(self.logdir) + @mason.util.job_step def _deploy_and_test_systems(self): config = self.plugin_config['config'] infrastructure = config['test-infrastructure-type'] build_test_config = (config['trove-host'], config['trove-id'], - config['build-ref-prefix']) - cluster = self.morph_helper.load_morphology( - config['cluster-morphology']) - systems = dict(morph_helper.load_cluster_systems(cluster)) + config.get('build-ref-prefix')) + cluster_path = config['cluster-morphology'] + cluster = self.morph_helper.load_morphology(cluster_path) + systems = dict(self.morph_helper.load_cluster_systems(cluster)) + + deployment_hosts = {} + for host_config in config['deployment-host']: + arch, address = host_config.split(':', 1) + user, address = address.split('@', 1) + address, disk_path = address.split(':', 1) + if user == '': + user = 'root' + deployment_hosts[arch] = mason.util.VMHost( + user, address, disk_path) for system_path, deployment_name, deployment_config in \ self.morph_helper.iterate_cluster_deployments(cluster): @@ -134,15 +177,30 @@ class Runner(mason.util.JobRunner): continue host = deployment_hosts[system['arch']] - deployment = mason.util.Deployment(cluster, deployment_name, - deployment_config, host) - - instance = deployment.deploy() + log_path = os.path.join(self.logdir, + '%s-deploy.log' % system['name']) + net_id = config['openstack-network-id'] + deployment = mason.deployment.Deployment(cluster_path, + deployment_name, + deployment_config, host, + net_id, log_path) + + os.chdir(self.defs_checkout) + instance = deployment.deploy(infrastructure) + log_path = os.path.join(self.logdir, + '%s-test.log' % system['name']) + logfile = open(log_path, 'w+') try: self._run_tests(instance, system_path, system, - build_test_config, systems) + build_test_config, systems, logfile) finally: instance.delete() + logfile.close() + + # If we cancel, we don't want to have to wait for all + # the systems to be tested before we stop. + self._handle_cancellation() + @mason.util.job_step def _clean_up(self): cliapp.runcmd(['rm', '-rf', self.workspace]) -- cgit v1.2.1