diff options
author | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2014-03-26 13:46:51 +0000 |
---|---|---|
committer | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2014-03-26 13:46:51 +0000 |
commit | 9d33544d23d9608a86858cfad384bb7501718f79 (patch) | |
tree | cacfa3208564417e1c376e68dd7b25542400fa01 | |
parent | 5afef32028fbcacb7acc3d0318407169dd2353de (diff) | |
download | lorry-9d33544d23d9608a86858cfad384bb7501718f79.tar.gz |
Remove original lorry binary
-rwxr-xr-x | lorry | 511 |
1 files changed, 0 insertions, 511 deletions
@@ -1,511 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import json -import logging -import os -import urllib2 -import string -import sys -from datetime import datetime -import shutil -import traceback - -__version__ = '0.0' - -lorry_path = os.path.realpath(__file__) - -def quote_url(url): - ''' Convert URIs to strings that only contain digits, letters, % and _. - - NOTE: When changing the code of this function, make sure to also apply - the same to the quote_url() function of morph. Otherwise the git bundles - generated by lorry may no longer be found by morph. - - ''' - valid_chars = string.digits + string.letters + '%_' - transl = lambda x: x if x in valid_chars else '_' - return ''.join([transl(x) for x in url]) - - -class Lorry(cliapp.Application): - - def add_settings(self): - self.settings.string(['working-area', 'w'], - 'use DIR as the working area (for holding ' - 'intermediate git repositories, etc)', - metavar='DIR', default='workd') - self.settings.string(['mirror-base-url-push'], - 'base URL to use for pushing to the mirror ' - 'server (default: %default)', - metavar='URL', - default='ssh://gitano@roadtrain.codethink.co.uk/delta') - self.settings.string(['mirror-base-url-fetch'], - 'base URL to use for bundle names and for ' - 'pulling from the mirror server (default: ' - '%default)', - metavar='URL', - default='git://git.baserock.org/delta') - self.settings.boolean(['pull-only'], - 'only pull from upstreams, do not push to ' - 'the mirror server') - self.settings.boolean(['verbose', 'v'], - 'report what is going on to stdout') - self.settings.boolean(['repack'], - 'repack git repositories when an import has ' - 'been updated (enabled by default)', default=True) - self.settings.string(['command-stdout'], - 'write the stdout of commands to this file', - metavar='FILE', default=None) - self.settings.string(['command-stderr'], - 'write the stderr of commands to this file', - metavar='FILE', default=None) - self.settings.choice(['bundle'], ['first', 'never', 'always'], - 'create bundles of git repositories.' - 'first will only bundle if there is not already ' - 'a bundle in BUNDLES (default: first)') - self.settings.string(['bundle-dest'], - 'put created bundles in BUNDLES', - metavar='BUNDLES') - self.settings.choice(['tarball'], ['first', 'never', 'always'], - 'create tarballs of git repositories.' - 'first will only tar if there is not already ' - 'a tarball in TARBALLS (default: first)') - self.settings.string(['tarball-dest'], - 'put created tarballs in TARBALLS', - metavar='TARBALLS') - self.settings.boolean(['keep-multiple-backups'], - 'keep multiple (time-stamped) backups (disabled by default)', - default=False) - - def process_args(self, args): - status = 0 - - self.settings['working-area'] = os.path.abspath(self.settings['working-area']) - if not os.path.exists(self.settings['working-area']): - os.makedirs(self.settings['working-area']) - - for arg in args: - self.progress('Processing spec file %s' % arg) - with open(arg) as f: - specs = json.load(f) - for name in sorted(specs.keys()): - self.progress('Getting: %s' % name) - try: - self.gitify(name, specs[name]) - except Exception,e: - status += 1 - sys.stderr.write( - 'Error mirroring:\n%s' % traceback.format_exc()) - logging.error(traceback.format_exc()) - if status > 0 : - logging.debug('Total Mirrors failed: %d' %status) - status = 1 - - self.progress('Done') - #print 'total failed:',status - sys.exit(status) - - def bundle(self, name, gitdir): - if self.settings['bundle'] == 'never': return - bundlename = "%s/%s" % (self.settings['mirror-base-url-fetch'], name) - path = os.path.join(self.settings['bundle-dest'], - quote_url(bundlename)) + '.bndl' - if not os.path.exists(path) or self.settings['bundle'] == 'always': - self.progress('.. building bundle %s' % bundlename) - - # create the bundle - self.run_program(['git', 'bundle', 'create', path, '--branches', - '--tags'], cwd=gitdir) - - # FIXME this is a hack to avoid unrecognized headers in bundles, - # which happens with some repositories. See - # - # http://marc.info/?l=git&m=132992959317420&w=2 - # - # for more information. From the bundle's header section, the - # expression below will remove all garbage lines that appear - # between the first line (the bundle format meta comment) and - # the list of refs. - expr = '1,/^[0-9a-f]\{40\}/{ /^[0-9a-f]\{40\}/!{/^[^#]/d}}' - self.run_program(['sed', '-i', '-e', expr, path], cwd=gitdir) - - def make_tarball(self, name, gitdir): - if self.settings['tarball'] == 'never': return - tarballname = "%s/%s" % (self.settings['mirror-base-url-fetch'], - name) - path = os.path.join(self.settings['tarball-dest'], - quote_url(tarballname)) + '.tar' - if os.path.exists(os.path.join(gitdir, '.git')): - gitdir = os.path.join(gitdir, '.git') - if not os.path.exists(path) or self.settings['tarball'] == 'always': - self.progress('.. building tarball %s' % tarballname) - args = ['tar', 'cf', path] - if os.path.exists(os.path.join(gitdir, 'config')): - os.rename(os.path.join(gitdir, 'config'), - os.path.join(gitdir, 'config.lorrytmp')) - with open(os.path.join(gitdir, 'config'), 'w') as fh: - fh.write("""[core] - repositoryformatversion = 0 - filemode = true - bare = true -""") - for entry in ['HEAD', 'objects', 'refs', - 'info', 'packed-refs', 'config', 'branches', - 'description']: - if os.path.exists(os.path.join(gitdir, entry)): - args += [entry] - self.run_program(args, cwd=gitdir) - if os.path.exists(os.path.join(gitdir, 'config.lorrytmp')): - os.rename(os.path.join(gitdir, 'config.lorrytmp'), - os.path.join(gitdir, 'config')) - - def gitify(self, name, spec): - self.progress('Getting %s' % name) - table = { - 'bzr': self.gitify_bzr, - 'cvs': self.gitify_cvs, - 'git': self.mirror_git, - 'hg': self.gitify_hg, - 'svn': self.gitify_svn, - 'tarball': self.gitify_tarball, - } - vcstype = spec['type'] - if vcstype not in table: - raise cliapp.AppException('Unknown VCS type %s' % vcstype) - dirname = self.dirname(name) - if not os.path.exists(dirname): - os.mkdir(dirname) - gitdir = os.path.join(dirname, 'git') - - time = datetime.now().strftime('%F-%T') - - pre_update_name = 'git-pre-update' - pre_update_backup_suffix = (pre_update_name + '-' + time - if self.settings['keep-multiple-backups'] else pre_update_name) - - pre_update_backup_dir = os.path.join(dirname, pre_update_backup_suffix) - - post_fail_name = 'git-post-fail' - post_fail_backup_suffix = (post_fail_name + '-' + time - if self.settings['keep-multiple-backups'] else post_fail_name) - - post_fail_backup_dir = os.path.join(dirname, post_fail_backup_suffix) - - if not self.settings['keep-multiple-backups']: - # remove previous backups if they exist - if os.path.exists(pre_update_backup_dir): - shutil.rmtree(pre_update_backup_dir) - - if os.path.exists(post_fail_backup_dir): - shutil.rmtree(post_fail_backup_dir) - - backupdir = self.backup_gitdir(name, gitdir, pre_update_backup_dir) - - try: - self.needs_aggressive = False - table[vcstype](name, dirname, gitdir, spec) - if self.settings['repack']: - self.progress('.. repacking %s git repository' % name) - self.run_program(['git', 'config', - 'pack.windowMemory', '128M'], cwd=gitdir) - args = ['git', 'gc'] - if self.needs_aggressive: - args += ['--aggressive'] - self.run_program(args, cwd=gitdir) - - self.bundle(name, gitdir) - self.make_tarball(name, gitdir) - except: - if backupdir is not None: - faildir = self.backup_gitdir(name, gitdir, post_fail_backup_dir) - self.restore_backup(name, backupdir, gitdir) - self.output.write('Mirror of %s failed, state before mirror ' - 'is saved at %s and state after mirror is ' - 'saved at %s\n' % (name, backupdir, faildir)) - logging.debug('Mirror of %s failed, state before mirror ' - 'is saved at %s and state after mirror is ' - 'saved at %s\n', name, backupdir, faildir) - raise - - if not self.settings['pull-only']: - if 'refspecs' in spec: - self.push_to_mirror_server(name, gitdir, spec['refspecs']) - else: - self.push_to_mirror_server(name, gitdir) - - if backupdir is not None: - self.progress('.. removing %s git repository backup' % name) - shutil.rmtree(backupdir) - - def restore_backup(self, name, backupdir, gitdir): - self.progress('.. restoring %s good git repository' % name) - dotgit = os.path.join(gitdir, '.git') - if not os.path.exists(dotgit): - dotgit = gitdir - shutil.rmtree(dotgit) - self.copy_gitdir(backupdir, dotgit) - - def backup_gitdir(self, name, gitdir, backupdir): - dotgit = os.path.join(gitdir, '.git') - if not os.path.exists(dotgit): - dotgit = gitdir - - self.progress('.. backing up %s git repository to %s' % (name, backupdir)) - - return self.copy_gitdir(dotgit, backupdir) - - def copy_gitdir(self, source, dest): - if not os.path.exists(source): - return None - - # copy everything except the objects dir - def ignoreobjects(dirname, filenames): - if dirname.endswith(source): - return ['objects'] - return [] - shutil.copytree(source, dest, ignore=ignoreobjects) - - # hardlink the objects - sourceobjects = os.path.join(source, 'objects') - assert os.path.exists(sourceobjects), "No source objects" - objectspath = os.path.join(dest, 'objects') - os.mkdir(objectspath) - for dirpath, dirnames, filenames in os.walk(sourceobjects): - assert dirpath.startswith(sourceobjects) - # strip sourceobjects and / from relpath - relpath = dirpath[len(sourceobjects)+1:] - for dirname in dirnames: - os.mkdir(os.path.join(objectspath, relpath, dirname)) - for filename in filenames: - assert os.path.exists(os.path.join(objectspath, relpath)) - os.link(os.path.join(dirpath, filename), - os.path.join(objectspath, relpath, filename)) - return dest - - def mirror_git(self, project_name, dirname, gitdir, spec): - if not os.path.exists(gitdir): - self.progress('.. initialising git dir') - self.run_program(['git', 'init', '--bare', gitdir]) - self.progress('.. updating existing clone') - if 'remote.origin.fetch' in self.run_program(['git', 'config', '-l'], - cwd=gitdir): - self.run_program(['git', 'config', '--unset-all', - 'remote.origin.fetch'], cwd=gitdir) - self.run_program(['git', 'config', 'remote.origin.url', - spec['url']], cwd=gitdir) - self.run_program(['git', 'config', 'remote.origin.mirror', 'true'], - cwd=gitdir) - self.run_program(['git', 'config', '--add', 'remote.origin.fetch', - '+refs/heads/*:refs/heads/*'], cwd=gitdir) - self.run_program(['git', 'config', '--add', 'remote.origin.fetch', - '+refs/tags/*:refs/tags/*'], cwd=gitdir) - try: - self.run_program(['git', 'remote', 'update', 'origin', '--prune'], - cwd=gitdir) - except: - self.run_program(['git', 'remote', 'prune', 'origin'], - cwd=gitdir) - self.run_program(['git', 'remote', 'update', 'origin', '--prune'], - cwd=gitdir) - - def gitify_bzr(self, project_name, dirname, gitdir, spec): - bzrdir = os.path.join(dirname, 'bzr') - # check if repo exists - if not os.path.exists(bzrdir): - self.progress('.. creating bzr repository') - self.run_program(['bzr', 'init-repo', '--no-trees', bzrdir]) - - if not os.path.exists(gitdir): - self.progress('.. creating git repo') - os.mkdir(gitdir) - self.run_program(['git', 'init', '--bare', gitdir]) - self.needs_aggressive = True - - # branches are the listed branches, plus the branch specified in url - if 'branches' in spec: - branches = spec['branches'] - else: - branches = {} - if 'url' in spec: - branches['trunk'] = spec['url'] - logging.debug('all branches: %s' % repr(branches)) - - for branch, address in branches.iteritems(): - branchdir = os.path.join(bzrdir, branch) - if not os.path.exists(branchdir): - self.progress('.. doing initial bzr branch') - self.run_program(['bzr', 'branch', '--quiet', address, - branchdir]) - else: - self.progress('.. updating bzr branch') - self.run_program(['bzr', 'pull', '--quiet', address], - cwd=branchdir) - - exports = {} - bzrmarks = os.path.join(gitdir, 'marks.bzr') - for branch, address in branches.iteritems(): - branchdir = os.path.join(bzrdir, branch) - self.progress('.. fast-exporting branch %s from bzr' % branch) - exports[branch] = os.path.join(dirname, 'fast-export' + branch) - cmdline = ['bzr', 'fast-export', '--git-branch=' + branch, - branchdir, exports[branch]] - if os.path.exists(bzrmarks): - cmdline.append('--marks=' + bzrmarks) - else: - cmdline.append('--export-marks=' + bzrmarks) - self.run_program(cmdline) - - gitmarks = os.path.join(gitdir, 'marks.git') - for branch, address in branches.iteritems(): - self.progress('.. fast-importing branch %s into git' % branch) - with open(exports[branch], 'rb') as exportfile: - cmdline = ['git', 'fast-import', '--export-marks=' + gitmarks] - if os.path.exists(gitmarks): - cmdline.append('--import-marks=' + gitmarks) - self.run_program(cmdline, stdin=exportfile, - cwd=gitdir) - - for branch, address in branches.iteritems(): - branchdir = os.path.join(bzrdir, branch) - self.progress('.. removing temporary fast-export file ' + - exports[branch]) - os.remove(exports[branch]) - - def gitify_svn(self, project_name, dirname, gitdir, spec): - if not os.path.exists(gitdir): - self.progress('.. doing initial clone') - self.needs_aggressive = True - layout = spec["layout"] - # if standard layour specified, fill in the defaults - if layout == "standard": - layout = { "trunk": "trunk", - "tags": "tags/*", - "branches": "branches/*" } - # init the repo then manually set the refspecs to fetch into local - # git-svn can apparently provide better history tracking by - # fetching the root of the repository - # git-svn will convert branch, trunk and tag paths to allow this, - # but it is simpler to disable it and do it manually - self.run_program(['git', 'svn', 'init', spec['url'], gitdir + "-tmp", - '--svn-remote=svn', '--no-minimize-url']) - os.rename(os.path.join(gitdir + "-tmp", '.git'), gitdir) - os.rmdir(gitdir + "-tmp") - self.run_program(['git', 'config', 'core.bare', 'true'], - cwd=gitdir) - self.run_program(['git', 'config', 'svn-remote.svn.fetch', - layout["trunk"]+':refs/heads/master'], - cwd=gitdir) - self.run_program(['git', 'config', 'svn-remote.svn.branches', - layout["branches"] + ':refs/heads/*'], - cwd=gitdir) - self.run_program(['git', 'config', 'svn-remote.svn.tags', - layout["tags"] + ':refs/tags/*'], - cwd=gitdir) - else: - self.progress('.. updating existing clone') - # update the remote tracking branches - self.run_program(['git', 'svn', 'fetch'], cwd=gitdir) - - def gitify_cvs(self, project_name, dirname, gitdir, spec): - self.needs_aggressive = True - self.run_program(['git', 'cvsimport', '-a', '-d', spec['url'], - '-C', gitdir, spec['module']]) - - def gitify_hg(self, project_name, dirname, gitdir, spec): - hgdir = os.path.join(dirname, 'hg') - if os.path.exists(hgdir): - self.progress('.. updating hg branch') - self.run_program(['hg', 'pull', '--quiet', '--insecure'], cwd=hgdir) - else: - self.progress('.. doing initial hg branch') - self.run_program(['hg', 'clone', '--quiet', '--insecure', spec['url'], hgdir]) - - if not os.path.exists(gitdir): - self.needs_aggressive = True - self.run_program(['git', 'init', '--bare', gitdir]) - - self.progress('.. fast-exporting into git') - self.run_program(['hg-fast-export', '--quiet', '--force', '-r', '../hg'], - cwd=gitdir) - - def gitify_tarball(self, project_name, dirname, gitdir, spec): - url = spec['url'] - url_path = urllib2.urlparse.urlparse(url)[2] - basename = os.path.basename(url_path) - tardest = os.path.join(dirname, basename) - self.progress('.. checking if we need to fetch %s' % basename) - if not os.path.exists(tardest): - self.progress('.. attempting to fetch.') - with open(tardest, 'w') as tarfile: - urlfile = urllib2.urlopen(spec['url']) - tarfile.write(urlfile.read()) - urlfile.close() - else: - self.progress('.. no need to run, nothing to do') - return - if not os.path.exists(gitdir): - self.run_program(['git', 'init', '--bare', gitdir]) - cmdline = ["%s.tar-importer" % lorry_path, tardest] - self.run_program(cmdline, cwd=gitdir) - self.needs_aggressive = True - - - def push_to_mirror_server(self, name, gitdir, - pushrefspecs=['refs/heads/*:refs/heads/*', - 'refs/tags/*:refs/tags/*']): - pushurl = "%s/%s.git" % (self.settings['mirror-base-url-push'], name) - self.progress('.. pushing %s to mirror server %s' % (name, pushurl)) - self.run_program(['git', 'push', pushurl]+pushrefspecs, cwd=gitdir) - - def run_program(self, argv, **kwargs): - if self.settings['command-stdout']: - kwargs['stdout'] = open(self.settings['command-stdout'], 'a') - if self.settings['command-stderr']: - kwargs['stderr'] = open(self.settings['command-stderr'], 'a') - if 'stdin' not in kwargs: - kwargs['stdin'] = open('/dev/null', 'r') - logging.debug('Running: argv=%s kwargs=%s' % - (repr(argv), repr(kwargs))) - exit, out, err = self.runcmd_unchecked(argv, **kwargs) - logging.debug('Command: %s\nExit: %s\nStdout:\n%sStderr:\n%s' % - (argv, exit, self.indent(out or ''), - self.indent(err or ''))) - if exit != 0: - raise Exception('%s failed (exit code %s):\n%s' % - (' '.join(argv), exit, self.indent(err or ''))) - return out - - def indent(self, string): - return ''.join(' %s\n' % line for line in string.splitlines()) - - def dirname(self, project_name): - assert '\0' not in project_name - # We escape slashes as underscores. - project_name = '_'.join(project_name.split('/')) - return os.path.join(self.settings['working-area'], project_name) - - def progress(self, msg): - logging.debug(msg) - if self.settings['verbose']: - self.output.write('%s\n' % msg) - - -if __name__ == '__main__': - Lorry(version=__version__).run() - |