path: root/lorry

#!/usr/bin/python
# Copyright (C) 2011  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import cliapp
import json
import logging
import os
import urllib2
import string
import sys

__version__ = '0.0'

def quote_url(url):
    ''' Convert URIs to strings that only contain digits, letters, % and _.

    NOTE: When changing the code of this function, make sure to also apply
    the same to the quote_url() function of morph. Otherwise the git bundles
    generated by lorry may no longer be found by morph.

    ''' 
    valid_chars = string.digits + string.letters + '%_'
    transl = lambda x: x if x in valid_chars else '_'
    return ''.join([transl(x) for x in url])


class Lorry(cliapp.Application):

    def add_settings(self):
        self.settings.string(['working-area', 'w'],
                             'use DIR as the working area (for holding '
                                'intermediate git repositories, etc)',
                             metavar='DIR')
        self.settings.string(['mirror-base-url-push'],
                             'base URL to use for pushing to the mirror '
                             'server (default: %default)',
                             metavar='URL',
                             default='ssh://git@gitorious.org:baserock-morphs')
        self.settings.string(['mirror-base-url-fetch'],
                             'base URL to use for bundle names and for '
                             'pulling from the mirror server (default: '
                             '%default)',
                             metavar='URL',
                             default='git://gitorious.org/baserock-morphs')
        self.settings.boolean(['pull-only'], 
                              'only pull from upstreams, do not push to '
                              'the mirror server')
        self.settings.boolean(['verbose', 'v'], 
                              'report what is going on to stdout')
        self.settings.boolean(['repack'], 
                              'repack git repositories when an import has'
                              'been updated (default: %default)', default=True)
        self.settings.string(['command-stdout'],
                             'write the stdout of commands to this file',
                             metavar='FILE', default=None)
        self.settings.string(['command-stderr'],
                             'write the stderr of commands to this file',
                             metavar='FILE', default=None)
        self.settings.choice(['bundle'], ['first', 'never', 'always'],
                             'create bundles of git repositories.'
                             'first will only bundle if there is not already a '
                             'bundle in BUNDLES (default: first)')
        self.settings.string(['bundle-dest'],
                             'put created bundles in BUNDLES',
                             metavar='BUNDLES')

    def process_args(self, args):
        status = 0
        
        if self.settings['working-area'] == '' :
            print 'No working area defined going to use "workd" directory'
            self.settings['working-area'] = 'workd'
        if not os.path.exists(self.settings['working-area']): 
            os.makedirs(self.settings['working-area'])

        for arg in args:
            self.progress('Processing spec file %s' % arg)
            with open(arg) as f:
                specs = json.load(f)
            for name in sorted(specs.keys()):
               try:
                    print 'Getting: ',name
                    self.gitify(name, specs[name])
               except Exception,e:
                    status += 1
                    print 'Exception:',e
                    logging.debug('Mirroring Error: %s' %e)
        if status > 0 :
           logging.debug('Total Mirrors failed: %d' %status)
           status = 1
 
        self.progress('Done')
        #print 'total failed:',status
        sys.exit(status) 

    def bundle(self, name, gitdir):
        if self.settings['bundle'] == 'never': return
        bundlename = "%s/%s" % (self.settings['mirror-base-url-fetch'], name)
        path = os.path.join(self.settings['bundle-dest'],
                            quote_url(bundlename))
        if not os.path.exists(path) or self.settings['bundle'] == 'always':
            self.progress('.. building bundle %s' % bundlename)
            self.run_program(['git', 'bundle', 'create', path, '--branches',
                              '--tags'], cwd=gitdir)

    def gitify(self, name, spec):
        self.progress('Getting %s' % name)
        table = {
            'bzr': self.gitify_bzr,
            'cvs': self.gitify_cvs,
            'git': self.mirror_git,
            'hg':  self.gitify_hg,
            'svn': self.gitify_svn,
            'tarball': self.gitify_tarball,
        }
        vcstype = spec['type']
        if vcstype not in table:
            raise cliapp.AppException('Unknown VCS type %s' % vcstype)
        dirname = self.dirname(name)
        if not os.path.exists(dirname):
            os.mkdir(dirname)
        gitdir = os.path.join(dirname, 'git')
        table[vcstype](name, dirname, gitdir, spec)
        if self.settings['repack']:
            self.progress('.. repacking %s git repository' % name)
            self.run_program(['git', 'repack', '-a', '-d', '--depth=250',
                              '--window=250'], cwd=gitdir)
        self.bundle(name, gitdir)
        if not self.settings['pull-only']:
            self.push_to_mirror_server(gitdir)
 
    def add_remote(self, name, gitdir,
                   pushrefspec=['refs/heads/*:refs/heads/*',
                                'refs/tags/*:refs/tags/*']):

        ''' Add mirror server as a remote to gitdir.
        
        Can optionally set up refspecs for pushing.
        
        '''

        self.progress('.. adding remote %s to %s' % ('gitorious', name))
        fetch_url = ('%s/%s.git' %
                     (self.settings['mirror-base-url-fetch'], name))
        push_url = ('%s/%s.git' %
                    (self.settings['mirror-base-url-push'], name))
        self.progress('.. will fetch from %s and push to %s' %
                      (fetch_url, push_url))
        self.run_program(['git', 'remote', 'add', 'gitorious', fetch_url],
                         cwd=gitdir)
        self.run_program(['git', 'remote', 'set-url', '--push', 'gitorious',
                          push_url], cwd=gitdir)
        if len(pushrefspec) >= 1:
            self.run_program(['git', 'config', 'remote.gitorious.push',
                              pushrefspec[0]], cwd=gitdir)
        for refspec in pushrefspec[1:]:
            self.run_program(['git', 'config', '--add', 'remote.gitorious.push',
                              refspec], cwd=gitdir)

    def mirror_git(self, project_name, dirname, gitdir, spec):
        if not os.path.exists(gitdir):
            self.progress('.. doing initial clone')
            self.run_program(['git', 'clone', '--mirror', spec['url'], gitdir])
            # init gitorious remote
            self.add_remote(project_name, gitdir)
        else:
            self.progress('.. updating existing clone')
            self.run_program(['git', 'fetch', 'origin'], cwd=gitdir)

    def gitify_bzr(self, project_name, dirname, gitdir, spec):
        bzrdir = os.path.join(dirname, 'bzr')
        # check if repo exists
        if not os.path.exists(bzrdir):
            self.progress('.. creating bzr repository')
            self.run_program(['bzr', 'init-repo', '--no-trees', bzrdir])

        if not os.path.exists(gitdir):
            self.progress('.. creating git repo')
            os.mkdir(gitdir)
            self.run_program(['git', 'init', gitdir])
            self.add_remote(project_name, gitdir)

        # branches are the listed branches, plus the branch specified in url
        if 'branches' in spec:
            branches = spec['branches']
        else:
            branches = {}
        if 'url' in spec:
            branches['trunk'] = spec['url']
        logging.debug('all branches: %s' % repr(branches))

        for branch, address in branches.iteritems():
            branchdir = os.path.join(bzrdir, branch)
            if not os.path.exists(branchdir):
                self.progress('.. doing initial bzr branch')
                self.run_program(['bzr', 'branch', '--quiet', address,
                                  branchdir])
            else:
                self.progress('.. updating bzr branch')
                self.run_program(['bzr', 'pull', '--quiet', address],
                                 cwd=branchdir)

        exports = {}
        bzrmarks = os.path.join(dirname, 'marks.bzr')
        for branch, address in branches.iteritems():
            branchdir = os.path.join(bzrdir, branch)
            self.progress('.. fast-exporting branch %s from bzr' % branch)
            exports[branch] = os.path.join(dirname, 'fast-export' + branch)
            cmdline = ['bzr', 'fast-export', '--git-branch=' + branch, branchdir, exports[branch]]
            if os.path.exists(bzrmarks):
                cmdline.append('--marks=' + bzrmarks)
            else:
                cmdline.append('--export-marks=' + bzrmarks)
            self.run_program(cmdline)

        gitmarks = os.path.join(dirname, 'marks.git')
        for branch, address in branches.iteritems():
            self.progress('.. fast-importing branch %s into git' % branch)
            with open(exports[branch], 'rb') as exportfile:
                cmdline = ['git', 'fast-import', '--export-marks=' + gitmarks]
                if os.path.exists(gitmarks):
                    cmdline.append('--import-marks=' + gitmarks)
                self.run_program(cmdline, stdin=exportfile,
                                 cwd=gitdir)

        for branch, address in branches.iteritems():
            branchdir = os.path.join(bzrdir, branch)
            self.progress('.. removing temporary fast-export file ' + exports[branch])
            os.remove(exports[branch])

    def gitify_svn(self, project_name, dirname, gitdir, spec):
        if not os.path.exists(gitdir):
            self.progress('.. doing initial clone')
            os.mkdir(gitdir)
            layout = spec["layout"]
            # if standard layour specified, fill in the defaults
            if layout == "standard":
                layout = { "trunk": "trunk",
                           "tags": "tags/*",
                           "branches": "branches/*" }
            # init the repo then manually set the refspecs to fetch into local
            # git-svn can apparently provide better history tracking by fetching
            # the root of the repository
            # git-svn will convert branch, trunk and tag paths to allow this,
            # but it is simpler to disable it and do it manually
            self.run_program(['git', 'svn', 'init', spec['url'], gitdir,
                              '--svn-remote=svn', '--no-minimize-url'])
            self.run_program(['git', 'config', 'svn-remote.svn.fetch',
                              layout["trunk"]+':refs/heads/master'], cwd=gitdir)
            self.run_program(['git', 'config', 'svn-remote.svn.branches',
                              layout["branches"] + ':refs/heads/*'], cwd=gitdir)
            self.run_program(['git', 'config', 'svn-remote.svn.tags',
                              layout["tags"] + ':refs/tags/*'],
                             cwd=gitdir)
            self.add_remote(project_name, gitdir)
        else:
            self.progress('.. updating existing clone')
        # update the remote tracking branches
        self.run_program(['git', 'svn', 'fetch'], cwd=gitdir)

    def gitify_cvs(self, project_name, dirname, gitdir, spec):
        self.run_program(['git', 'cvsimport', '-a', '-d', spec['url'],
                          '-C', gitdir, spec['module']])
        out = self.run_program(['git', 'remote'], cwd=gitdir)
        if 'gitorious' not in out.splitlines():
            self.add_remote(project_name, gitdir)

    def gitify_hg(self, project_name, dirname, gitdir, spec):
        hgdir = os.path.join(dirname, 'hg')
        if os.path.exists(hgdir):
            self.progress('.. updating hg branch')
            self.run_program(['hg', 'pull', '--quiet'], cwd=hgdir)
        else:
            self.progress('.. doing initial hg branch')
            self.run_program(['hg', 'clone', '--quiet', spec['url'], hgdir])

        if not os.path.exists(gitdir):
            self.run_program(['git', 'init', gitdir])
            self.add_remote(project_name, gitdir)
        
        self.progress('.. fast-exporting into git')
        self.run_program(['hg-fast-export', '--quiet', '-r', '../hg'], 
                         cwd=gitdir)

    def gitify_tarball(self, project_name, dirname, gitdir, spec):
        tardest = os.path.join(dirname, 'tarball')
        if not os.path.exists(tardest):
            with open(tardest, 'w') as tarfile:
                urlfile = urllib2.urlopen(spec['url'])
                tarfile.write(urlfile.read())
                urlfile.close()

        if not os.path.exists(gitdir):
            self.run_program(['git', 'init', gitdir])
            cmdline = ['tar', '--extract', '--file', tardest]
            # compression is handled in long form, so use gzip instead of z
            try:
                cmdline += ['--' + spec['compression']]
            except KeyError:
                pass
            # tarballs often have a directory on top, strip = 1 will remove it
            try:
                cmdline += ['--strip-components', str(spec['strip'])]
            except KeyError:
                pass
            self.run_program(cmdline, cwd=gitdir)
            self.run_program(['git', 'add', '.'], cwd=gitdir)
            self.run_program(['git', 'commit', '-m', 'Tarball conversion'],
                             cwd=gitdir)
            self.add_remote(project_name, gitdir)
            

    def push_to_mirror_server(self, gitdir):
        self.progress('.. pushing to mirror server')
        self.run_program(['git', 'push', 'gitorious'], cwd=gitdir)

    def run_program(self, argv, **kwargs):
        if self.settings['command-stdout']:
            kwargs['stdout'] = open(self.settings['command-stdout'], 'a')
        if self.settings['command-stderr']:
            kwargs['stderr'] = open(self.settings['command-stderr'], 'a')
        logging.debug('Running: argv=%s kwargs=%s' % 
                        (repr(argv), repr(kwargs)))
        exit, out, err = self.runcmd_unchecked(argv, **kwargs)
        logging.debug('Command: %s\nExit: %s\nStdout:\n%sStderr:\n%s' % 
                      (argv, exit, self.indent(out or ''),
                       self.indent(err or '')))
        if exit != 0:
            raise Exception('%s failed (exit code %s):\n%s' % 
                                (' '.join(argv), exit, self.indent(err or '')))
        return out

    def indent(self, string):
        return ''.join('    %s\n' % line for line in string.splitlines())
    
    def dirname(self, project_name):
        assert '/' not in project_name
        assert '\0' not in project_name
        return os.path.join(self.settings['working-area'], project_name)

    def progress(self, msg):
        logging.debug(msg)
        if self.settings['verbose']:
            self.output.write('%s\n' % msg)


if __name__ == '__main__':
    Lorry(version=__version__).run()