path: root/lorry

#!/usr/bin/python
# Copyright (C) 2011  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import cliapp
import json
import logging
import os
import urllib2


__version__ = '0.0'


class Lorry(cliapp.Application):

    def add_settings(self):
        self.settings.string(['working-area', 'w'],
                             'use DIR as the working area (for holding '
                                'intermediate git repositories, etc)',
                             metavar='DIR')
        self.settings.string(['gitorious-base-url'],
                             'prefix project names with URL when constructing '
                                'the git URL on Gitorious (default: %default)',
                             metavar='URL',
                             default='git@gitorious.org:baserock-morphs')
        self.settings.boolean(['pull-only'], 
                              'only pull from upstreams, do not push to '
                                'gitorious')
        self.settings.boolean(['verbose', 'v'], 
                              'report what is going on to stdout')
        self.settings.boolean(['repack'], 
                              'repack git repositories when an import has'
                              'been updated (default: %default)', default=True)

    def process_args(self, args):
        for arg in args:
            self.progress('Processing spec file %s' % arg)
            with open(arg) as f:
                specs = json.load(f)
            for name in sorted(specs.keys()):
                self.gitify(name, specs[name])
        self.progress('Done')
    
    def gitify(self, name, spec):
        self.progress('Getting %s' % name)
        table = {
            'bzr': self.gitify_bzr,
            'cvs': self.gitify_cvs,
            'git': self.mirror_git,
            'hg':  self.gitify_hg,
            'svn': self.gitify_svn,
            'tarball': self.gitify_tarball,
        }
        vcstype = spec['type']
        if vcstype not in table:
            raise cliapp.AppException('Unknown VCS type %s' % vcstype)
        dirname = self.dirname(name)
        if not os.path.exists(dirname):
            os.mkdir(dirname)
        gitdir = os.path.join(dirname, 'git')
        table[vcstype](dirname, gitdir, spec)
        if self.settings['repack']:
            self.progress('.. repacking %s git repository' % name)
            self.run_program(['git', 'repack', '-a', '-d', '--depth=250',
                              '--window=250'], cwd=gitdir)
        if not self.settings['pull-only']:
            self.push_to_gitorious(name, gitdir)
        
    def mirror_git(self, dirname, gitdir, spec):
        if not os.path.exists(gitdir):
            self.progress('.. doing initial clone')
            self.run_program(['git', 'clone', '--mirror', spec['url'], gitdir])
        else:
            self.progress('.. updating existing clone')
            self.run_program(['git', 'fetch', '--all'], cwd=gitdir)

    def gitify_bzr(self, dirname, gitdir, spec):
        bzrdir = os.path.join(dirname, 'bzr')
        # check if repo exists
        if not os.path.exists(bzrdir):
            self.progress('.. creating bzr repository')
            self.run_program(['bzr', 'init-repo', '--no-trees', bzrdir])

        if not os.path.exists(gitdir):
            self.progress('.. creating git repo')
            os.mkdir(gitdir)
            self.run_program(['git', 'init', gitdir])

        # branches are the listed branches, plus the branch specified in url
        if 'branches' in spec:
            branches = spec['branches']
        else:
            branches = {}
        if 'url' in spec:
            #branches.append(spec['url'])
            branches['trunk'] = spec['url']
        logging.debug('all branches: %s' % repr(branches))

        for branch, address in branches.iteritems():
            branchdir = os.path.join(bzrdir, branch)
            if not os.path.exists(branchdir):
                self.progress('.. doing initial bzr branch')
                self.run_program(['bzr', 'branch', '--quiet', address,
                                  branchdir])
            else:
                self.progress('.. updating bzr branch')
                self.run_program(['bzr', 'pull', '--quiet', address],
                                 cwd=branchdir)

        exports = {}
        bzrmarks = os.path.join(dirname, 'marks.bzr')
        for branch, address in branches.iteritems():
            branchdir = os.path.join(bzrdir, branch)
            self.progress('.. fast-exporting branch %s from bzr' % branch)
            exports[branch] = os.path.join(dirname, 'fast-export' + branch)
            cmdline = ['bzr', 'fast-export', '--git-branch=' + branch, branchdir, exports[branch]]
            if os.path.exists(bzrmarks):
                cmdline.append('--marks=' + bzrmarks)
            else:
                cmdline.append('--export-marks=' + bzrmarks)
            self.run_program(cmdline)

        gitmarks = os.path.join(dirname, 'marks.git')
        for branch, address in branches.iteritems():
            self.progress('.. fast-importing branch %s into git' % branch)
            with open(exports[branch], 'rb') as exportfile:
                cmdline = ['git', 'fast-import', '--export-marks=' + gitmarks]
                if os.path.exists(gitmarks):
                    cmdline.append('--import-marks=' + gitmarks)
                self.run_program(cmdline, stdin=exportfile,
                                 cwd=gitdir)

        for branch, address in branches.iteritems():
            branchdir = os.path.join(bzrdir, branch)
            self.progress('.. removing temporary fast-export file ' + exports[branch])
            os.remove(exports[branch])

    def gitify_svn(self, dirname, gitdir, spec):
        if not os.path.exists(gitdir):
            self.progress('.. doing initial clone')
            os.mkdir(gitdir)
            self.run_program(['git', 'svn', 'clone', spec['url'], gitdir])
        else:
            self.progress('.. updating existing clone')
            self.run_program(['git', 'svn', 'fetch'], cwd=gitdir)

    def gitify_cvs(self, dirname, gitdir, spec):
        self.run_program(['git', 'cvsimport', '-d', spec['url'],
                          '-C', gitdir, spec['module']])

    def gitify_hg(self, dirname, gitdir, spec):
        hgdir = os.path.join(dirname, 'hg')
        if os.path.exists(hgdir):
            self.progress('.. updating hg branch')
            self.run_program(['hg', 'pull', '--quiet'], cwd=hgdir)
        else:
            self.progress('.. doing initial hg branch')
            self.run_program(['hg', 'clone', '--quiet', spec['url'], hgdir])

        if not os.path.exists(gitdir):
            self.run_program(['git', 'init', gitdir])
        
        self.progress('.. fast-exporting into git')
        self.run_program(['hg-fast-export', '--quiet', '-r', '../hg'], 
                         cwd=gitdir)

    def gitify_tarball(self, dirname, gitdir, spec):
        tardest = os.path.join(dirname, 'tarball')
        if not os.path.exists(tardest):
            with open(tardest, 'w') as tarfile:
                urlfile = urllib2.urlopen(spec['url'])
                tarfile.write(urlfile.read())
                urlfile.close()

        if not os.path.exists(gitdir):
            self.run_program(['git', 'init', gitdir])
            cmdline = ['tar', '--extract', '--file', tardest]
            # compression is handled in long form, so use gzip instead of z
            try:
                cmdline += ['--' + spec['compression']]
            except KeyError:
                pass
            # tarballs often have a directory on top, strip = 1 will remove it
            try:
                cmdline += ['--strip-components', str(spec['strip'])]
            except KeyError:
                pass
            self.run_program(cmdline, cwd=gitdir)
            self.run_program(['git', 'add', '.'], cwd=gitdir)
            self.run_program(['git', 'commit', '-m', 'Tarball conversion'],
                             cwd=gitdir)
            

    def push_to_gitorious(self, project_name, gitdir):
        out = self.run_program(['git', 'remote', 'show'], cwd=gitdir)
        if 'gitorious' not in out.splitlines():
            self.progress('.. adding gitorious as a remote')
            url = ('%s/%s.git' % 
                   (self.settings['gitorious-base-url'], project_name))
            self.run_program(['git', 'remote', 'add', 'gitorious', url],
                             cwd=gitdir)

        self.progress('.. pushing branches to gitorious')
        self.run_program(['git', 'push', '--all', 'gitorious'], cwd=gitdir)
        self.progress('.. pushing tags to gitorious')
        self.run_program(['git', 'push', '--tags', 'gitorious'], cwd=gitdir)

    def run_program(self, argv, **kwargs):
        logging.debug('Running: argv=%s kwargs=%s' % 
                        (repr(argv), repr(kwargs)))
        exit, out, err = self.runcmd_unchecked(argv, **kwargs)
        logging.debug('Command: %s\nExit: %s\nStdout:\n%sStderr:\n%s' % 
                      (argv, exit, self.indent(out), self.indent(err)))
        if exit != 0:
            raise Exception('%s failed (exit code %s):\n%s' % 
                                (' '.join(argv), exit, self.indent(err)))
        return out

    def indent(self, string):
        return ''.join('    %s\n' % line for line in string.splitlines())
    
    def dirname(self, project_name):
        assert '/' not in project_name
        assert '\0' not in project_name
        return os.path.join(self.settings['working-area'], project_name)

    def progress(self, msg):
        logging.debug(msg)
        if self.settings['verbose']:
            self.output.write('%s\n' % msg)


if __name__ == '__main__':
    Lorry(version=__version__).run()