Fix up Lorry to expect repositories to be bare.

This patch makes Lorry always create bare repositories where it can (Note that
it cannot for CVS imports) and to create tarballs of bare repositories (if not
disabled) which will be more efficient than bundles for creation and cloning.

We may be able to disable bundles later.

1 files changed, 63 insertions, 29 deletions

diff --git a/lorry b/lorry
index 29df6dd..39fed88 100755
--- a/lorry
+++ b/lorry
@@ -80,6 +80,13 @@ class Lorry(cliapp.Application):
         self.settings.string(['bundle-dest'],
                              'put created bundles in BUNDLES',
                              metavar='BUNDLES')
+        self.settings.choice(['tarball'], ['first', 'never', 'always'],
+                             'create tarballs of git repositories.'
+                             'first will only tar if there is not already '
+                             'a tarball in TARBALLS (default: first)')
+        self.settings.string(['tarball-dest'],
+                             'put created tarballs in TARBALLS',
+                             metavar='TARBALLS')
 
     def process_args(self, args):
         status = 0
@@ -133,6 +140,36 @@ class Lorry(cliapp.Application):
             expr = '1,/^[0-9a-f]\{40\}/{ /^[0-9a-f]\{40\}/!{/^[^#]/d}}'
             self.run_program(['sed', '-i', '-e', expr, path], cwd=gitdir)
 
+    def make_tarball(self, name, gitdir):
+        if self.settings['tarball'] == 'never': return
+        tarballname = "%s/%s" % (self.settings['mirror-base-url-fetch'],
+                                 name)
+        path = os.path.join(self.settings['tarball-dest'],
+                            quote_url(tarballname)) + '.tar'
+        if os.path.exists(os.path.join(gitdir, '.git')):
+            gitdir = os.path.join(gitdir, '.git')
+        if not os.path.exists(path) or self.settings['tarball'] == 'always':
+            self.progress('.. building tarball %s' % tarballname)
+            args = ['tar', 'cf', path]
+            if os.path.exists(os.path.join(gitdir, 'config')):
+                os.rename(os.path.join(gitdir, 'config'),
+                          os.path.join(gitdir, 'config.lorrytmp'))
+                with open(os.path.join(gitdir, 'config'), 'w') as fh:
+                    fh.write("""[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+""")
+            for entry in ['HEAD', 'objects', 'refs',
+                          'info', 'packed-refs', 'config', 'branches',
+                          'description']:
+                if os.path.exists(os.path.join(gitdir, entry)):
+                    args += [entry]
+            self.run_program(args, cwd=gitdir)
+            if os.path.exists(os.path.join(gitdir, 'config.lorrytmp')):
+                os.rename(os.path.join(gitdir, 'config.lorrytmp'),
+                          os.path.join(gitdir, 'config'))
+
     def gitify(self, name, spec):
         self.progress('Getting %s' % name)
         table = {
@@ -158,6 +195,7 @@ class Lorry(cliapp.Application):
                 self.run_program(['git', 'repack', '-a', '-d', '--depth=250',
                                   '--window=250'], cwd=gitdir)
             self.bundle(name, gitdir)
+            self.make_tarball(name, gitdir)
         except:
             if backupdir is not None:
                 faildir = self.save_failgit(name, dirname, gitdir)
@@ -238,13 +276,12 @@ class Lorry(cliapp.Application):
 
     def mirror_git(self, project_name, dirname, gitdir, spec):
         if not os.path.exists(gitdir):
-            self.progress('.. doing initial clone')
-            self.run_program(['git', 'clone', '--mirror', spec['url'], gitdir])
-        else:
-            self.progress('.. updating existing clone')
-            self.run_program(['git', 'fetch', spec['url'],
-                              '+refs/heads/*:refs/heads/*',
-                              '+refs/tags/*:refs/tags/*'], cwd=gitdir)
+            self.progress('.. initialising git dir')
+            self.run_program(['git', 'init', '--bare', gitdir])
+        self.progress('.. updating existing clone')
+        self.run_program(['git', 'fetch', spec['url'],
+                          '+refs/heads/*:refs/heads/*',
+                          '+refs/tags/*:refs/tags/*'], cwd=gitdir)
 
     def gitify_bzr(self, project_name, dirname, gitdir, spec):
         bzrdir = os.path.join(dirname, 'bzr')
@@ -256,7 +293,7 @@ class Lorry(cliapp.Application):
         if not os.path.exists(gitdir):
             self.progress('.. creating git repo')
             os.mkdir(gitdir)
-            self.run_program(['git', 'init', gitdir])
+            self.run_program(['git', 'init', '--bare', gitdir])
 
         # branches are the listed branches, plus the branch specified in url
         if 'branches' in spec:
@@ -279,7 +316,7 @@ class Lorry(cliapp.Application):
                                  cwd=branchdir)
 
         exports = {}
-        bzrmarks = os.path.join(gitdir, '.git', 'marks.bzr')
+        bzrmarks = os.path.join(gitdir, 'marks.bzr')
         for branch, address in branches.iteritems():
             branchdir = os.path.join(bzrdir, branch)
             self.progress('.. fast-exporting branch %s from bzr' % branch)
@@ -292,7 +329,7 @@ class Lorry(cliapp.Application):
                 cmdline.append('--export-marks=' + bzrmarks)
             self.run_program(cmdline)
 
-        gitmarks = os.path.join(gitdir, '.git', 'marks.git')
+        gitmarks = os.path.join(gitdir, 'marks.git')
         for branch, address in branches.iteritems():
             self.progress('.. fast-importing branch %s into git' % branch)
             with open(exports[branch], 'rb') as exportfile:
@@ -323,8 +360,12 @@ class Lorry(cliapp.Application):
             # fetching the root of the repository
             # git-svn will convert branch, trunk and tag paths to allow this,
             # but it is simpler to disable it and do it manually
-            self.run_program(['git', 'svn', 'init', spec['url'], gitdir,
+            self.run_program(['git', 'svn', 'init', spec['url'], gitdir + "-tmp",
                               '--svn-remote=svn', '--no-minimize-url'])
+            os.rename(os.path.join(gitdir + "-tmp", '.git'), gitdir)
+            os.rmdir(gitdir + "-tmp")
+            self.run_program(['git', 'config', 'core.bare', 'true'],
+                             cwd=gitdir)
             self.run_program(['git', 'config', 'svn-remote.svn.fetch',
                               layout["trunk"]+':refs/heads/master'], 
                              cwd=gitdir)
@@ -353,37 +394,30 @@ class Lorry(cliapp.Application):
             self.run_program(['hg', 'clone', '--quiet', spec['url'], hgdir])
 
         if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', gitdir])
+            self.run_program(['git', 'init', '--bare', gitdir])
         
         self.progress('.. fast-exporting into git')
         self.run_program(['hg-fast-export', '--quiet', '-r', '../hg'], 
                          cwd=gitdir)
 
     def gitify_tarball(self, project_name, dirname, gitdir, spec):
-        tardest = os.path.join(dirname, 'tarball')
+        url = spec['url']
+        url_path = urllib2.urlparse.urlparse(url)[2]
+        basename = os.path.basename(url_path)
+        tardest = os.path.join(dirname, basename)
+        self.progress('.. checking if we need to fetch %s' % basename)
         if not os.path.exists(tardest):
+            self.progress('.. attempting to fetch.')
             with open(tardest, 'w') as tarfile:
                 urlfile = urllib2.urlopen(spec['url'])
                 tarfile.write(urlfile.read())
                 urlfile.close()
-
+        else:
+            self.progress('.. no need to run, nothing to do')
         if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', gitdir])
-        cmdline = ['tar', '--extract', '--file', tardest]
-        # compression is handled in long form, so use gzip instead of z
-        try:
-            cmdline += ['--' + spec['compression']]
-        except KeyError:
-            pass
-        # tarballs often have a directory on top, strip = 1 will remove it
-        try:
-            cmdline += ['--strip-components', str(spec['strip'])]
-        except KeyError:
-            pass
+            self.run_program(['git', 'init', '--bare', gitdir])
+        cmdline = ["%s.tar-importer" % __file__, tardest]
         self.run_program(cmdline, cwd=gitdir)
-        self.run_program(['git', 'add', '.'], cwd=gitdir)
-        self.run_program(['git', 'commit', '-m', 'Tarball conversion'],
-                         cwd=gitdir)
             
 
     def push_to_mirror_server(self, name, gitdir,