From 3b8e20ae645a0a8f6b57c053f4598407edce1d36 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 15 Jun 2015 17:32:24 +0100 Subject: Add script to submit builds of artifacts to the server --- scripts/submit-build | 220 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100755 scripts/submit-build diff --git a/scripts/submit-build b/scripts/submit-build new file mode 100755 index 0000000..7da0750 --- /dev/null +++ b/scripts/submit-build @@ -0,0 +1,220 @@ +#!/usr/bin/env python +# +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +'''Submit information on builds of Baserock artifacts. + +Usage: submit-build --host example.com --builder-name "Bob T. Builder" FILE [...] + +''' + + +import requests + +import argparse +import hashlib +import logging +import os +import re +import sys +import tarfile + + +class MetadataError(RuntimeError): + pass + + +def parse_args(): + parser = argparse.ArgumentParser( + description='submit builds to morph-cache-server') + parser.add_argument('--builder-name', '-b', type=str, required=True) + parser.add_argument('--host', type=str) + parser.add_argument('--url', type=str) + + parser.add_argument('files', metavar='files', nargs='+', type=str) + + args = parser.parse_args() + + if args.url and args.host: + raise RuntimeError("Please only specify one of --host and --url.") + + if not args.url and not args.host: + raise RuntimeError("Please specify one of --host and --url.") + + if args.url is None and args.host: + url = 'http://%s:8080/' % args.host + else: + url = args.url + + return args.files, url, args.builder_name + + +# Morph artifacts are: SHA256.type.name +# YBD artifacts are: name.SHA256 +# This just checks that there's a SHA256 pattern in there somewhere. +artifact_name_pattern = re.compile('.*[a-f0-9]{64}.*') + + +def filename_looks_like_a_baserock_artifact(filename): + '''Rough heuristic to determine if a file is a Morph or YBD artifact. + + This is just to avoid having lots of accidental submissions of files that + obviously aren't build artifacts. + + FIXME: the name logic should be server-side, not client-side! + + ''' + match = artifact_name_pattern.match(os.path.basename(filename)) + return (match is not None) + + +def parse_ybd_artifact_metadata(text): + lines = text.splitlines() + + if len(lines) < 2: + raise MetadataError("Not enough lines in metadata.") + + repo_line, ref_line = lines[0:2] + + if repo_line.startswith('repo:'): + repo = repo_line[6:] + else: + raise MetadataError("Expected 'repo:' on first line.") + + if ref_line.startswith('ref:'): + ref = ref_line[5:] + else: + raise MetadataError("Expected 'ref:' on second line.") + + # FIXME: this feels like a big hack. I think YBD should put the actual URL + # in the metadata. + if repo.startswith('baserock:'): + repo = 'git://git.baserock.org/baserock/' + repo[9:] + if repo.startswith('upstream:'): + repo = 'git://git.baserock.org/delta/' + repo[9:] + + return repo, ref + + +def read_metadata_from_artifact(artifact_tarfile): + members = artifact_tarfile.getmembers() + metadata_files = sorted(t for t in members if + t.name.startswith('./baserock')) + + if len(metadata_files) == 0: + raise MetadataError( + "No ./baserock entry in tarfile; this seems not to be a Baserock " + "artifact.") + elif len(metadata_files) == 1: + raise MetadataError( + "Found %s in tarfile, but expected a ./baserock directory and " + "a .meta file.", metadata_files[0].name) + elif len(metadata_files) == 2: + # Because we used sorted() above, the ./baserock directory will be + # first, and the .meta file second. + metadata_file = metadata_files[1] + else: + # It mostly ignores system artifacts because I'm not sure how to know + # which metadata file is the right one... + raise MetadataError( + "Looks like a system artifact, ignoring it.") + + # FIXME: support Morph metadata too + f = artifact_tarfile.extractfile(metadata_file) + if f is None: + raise MetadataError( + "Unable to read %s from artifact." % metadata_file.name) + metadata_text = f.read() + + repo, ref = parse_ybd_artifact_metadata(metadata_text) + + return repo, ref + + +def checksum(filename, hasher): + with open(filename,'rb') as f: + for chunk in iter(lambda: f.read(128 * hasher.block_size), b''): + hasher.update(chunk) + return hasher.hexdigest() + + +def submit_build_info_for_file(url, builder_name, filename, source_repo=None, + source_ref=None): + log = logging.getLogger() + log.debug("Calculating SHA1 hash of %s", filename) + + # Flush the logger here because calculating the hash is pretty slow. + # If you are tailing the log output it's nice to know what the program + # is doing. + for handler in log.handlers: + handler.flush() + + hash_sha1 = checksum(filename, hashlib.sha1()) + logging.debug("Checksum for %s: %s", filename, hash_sha1) + + ctime = os.stat(filename).st_ctime + logging.debug("Creation time of %s: %s", filename, ctime) + + logging.debug("Submitting to remote cache.") + params = { + 'cache_name': os.path.basename(filename), + 'builder_name': builder_name, + 'build_datetime': ctime, + 'hash_sha1': hash_sha1, + 'source_repo': source_repo, + 'source_ref': source_ref, + } + requests.put(url + '2.0/builds', params=params) + + +def run(): + files, cache_server_url, builder_name = parse_args() + + bad_files = [] + submitted_files = [] + for filename in files: + + if not os.path.isfile(filename): + sys.stderr.write("%s: not a file.\n" % filename) + bad_files.append(filename) + elif not filename_looks_like_a_baserock_artifact(filename): + sys.stderr.write( + "%s: No SHA256 hash in the filename, probably not a Baserock " + "build artifact.\n" % filename) + bad_files.append(filename) + elif not tarfile.is_tarfile(filename): + sys.stderr.write( + "%s: Not a valid tarfile, according to Python 'tarfile' " + "module.\n" % filename) + bad_files.append(filename) + else: + try: + with tarfile.open(filename) as f: + repo, ref = read_metadata_from_artifact(f) + submit_build_info_for_file( + cache_server_url, builder_name, filename, repo, ref) + submitted_files.append(filename) + except MetadataError as e: + sys.stderr.write("%s: %s\n" % (filename, e)) + + +if __name__ == '__main__': + try: + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + run() + except RuntimeError as e: + sys.stderr.write('ERROR: %s\n' % e) + sys.exit(1) -- cgit v1.2.1