Merge branch 'bwh/bare-working-repos' into 'master'

Ensure all working repositories are bare

Closes #12

See merge request CodethinkLabs/lorry/lorry!18

5 files changed, 123 insertions, 35 deletions

diff --git a/lorry b/lorry
index 4044e6d..524ed2c 100755
--- a/lorry
+++ b/lorry
@@ -423,6 +423,13 @@ class Lorry(cliapp.Application):
                 if not os.path.exists(old_gitdir):
                     old_gitdir = old_repo
 
+                # Ensure that it's bare
+                self.run_program(['git', 'config', 'core.bare', 'true'],
+                                 cwd=old_gitdir)
+                self.run_program(['git', 'config', 'core.logallrefupdates',
+                                  'false'],
+                                 cwd=old_gitdir)
+
                 self.write_update_count(old_gitdir, 1)
 
                 # Move it to new name, and remove top-level directory if we
@@ -471,6 +478,8 @@ class Lorry(cliapp.Application):
                 shutil.rmtree(temp_repo)
 
         if active_count == 0:
+            # We can't create the repo here because "git cvsimport"
+            # insists on doing so itself
             return temp_repo, None, 1
 
         self.copy_gitdir(active_repo, temp_repo)
@@ -481,6 +490,17 @@ class Lorry(cliapp.Application):
         with open(count_name, 'w') as count_file:
             count_file.write('%d\n' % count)
 
+    def ensure_gitdir(self, gitdir):
+        # Create git directory if it doesn't exist.  Return flag for
+        # whether we created it.
+        exists = os.path.exists(gitdir)
+        if not exists:
+            self.progress('.. creating git repo')
+            self.run_program(['git', 'init', '--bare', gitdir])
+        else:
+            self.progress('.. updating existing clone')
+        return not exists
+
     def copy_gitdir(self, source, dest):
         if not os.path.exists(source):
             return None
@@ -516,10 +536,7 @@ class Lorry(cliapp.Application):
             env = dict(os.environ)
             env['GIT_SSL_NO_VERIFY'] = 'true'
 
-        if not os.path.exists(gitdir):
-            self.progress('.. initialising git dir')
-            self.run_program(['git', 'init', '--bare', gitdir])
-        self.progress('.. updating existing clone')
+        self.ensure_gitdir(gitdir)
         argv = ['git', '-c', 'gc.autodetach=false', 'fetch', '--prune',
                 spec['url'], '+refs/heads/*:refs/heads/*',
                 '+refs/tags/*:refs/tags/*']
@@ -538,11 +555,7 @@ class Lorry(cliapp.Application):
             self.progress('.. creating bzr repository')
             self.run_program([bzr, 'init-repo', '--no-trees', bzrdir])
 
-        if not os.path.exists(gitdir):
-            self.progress('.. creating git repo')
-            os.mkdir(gitdir)
-            self.run_program(['git', 'init', '--bare', gitdir])
-            self.needs_aggressive = True
+        self.needs_aggressive = self.ensure_gitdir(gitdir)
 
         # branches are the listed branches, plus the branch specified in url
         if 'branches' in spec:
@@ -608,24 +621,22 @@ class Lorry(cliapp.Application):
             layout = { "trunk": "trunk",
                        "tags": "tags/*",
                        "branches": "branches/*" }
-        if not os.path.exists(gitdir):
-            self.progress('.. doing initial clone')
-            self.needs_aggressive = True
-            self.run_program(['git', 'svn', 'init', spec['url'], gitdir + "-tmp",
-                              '--svn-remote=svn', '--no-minimize-url'])
-            os.rename(os.path.join(gitdir + "-tmp", '.git'), gitdir)
-            os.rmdir(gitdir + "-tmp")
-            self.run_program(['git', 'config', 'core.bare', 'true'],
-                             cwd=gitdir)
-        else:
-            self.progress('.. updating existing clone')
 
-            # Force URL to the one in the Lorry spec. This way, if the
-            # URL in the spec changes, Lorry accepts the change rather
-            # than using the original one.
-            self.run_program(
-                ['git', 'config', 'svn-remote.svn.url', spec['url']],
-                cwd=gitdir)
+        # We should not run "git svn init" which creates a non-bare
+        # repository.  Instead, create the directory and extra config
+        # parameters that it would create.  This also ensures that if
+        # the URL in the spec changes, Lorry accepts the change rather
+        # than using the original one.
+        self.needs_aggressive = self.ensure_gitdir(gitdir)
+        os.makedirs(os.path.join(gitdir, 'svn/refs/remotes/git-svn'),
+                    exist_ok=True)
+        self.run_program(
+            ['git', 'config', 'svn-remote.svn.url', spec['url']],
+            cwd=gitdir)
+        self.run_program(
+            ['git', 'config', 'svn-remote.svn.fetch', ':refs/remotes/git-svn'],
+            cwd=gitdir)
+
         # manually set the refspecs to fetch into local
         # git-svn can apparently provide better history tracking by
         # fetching the root of the repository
@@ -668,11 +679,18 @@ class Lorry(cliapp.Application):
         self.needs_aggressive = True
         env = dict(os.environ)
         env['CVS_RSH'] = 'lorry-ssh-wrapper'
+        env['GIT_DIR'] = gitdir
         self.run_program(
-            ['git', 'cvsimport', '-a', '-d', spec['url'], 
-             '-C', gitdir, spec['module']],
+            ['git', 'cvsimport', '-a', '-d', spec['url'],
+             '-i', spec['module']],
             env=env)
 
+        # git cvsimport may create an index even in a bare repo
+        try:
+            os.remove(os.path.join(gitdir, 'index'))
+        except FileNotFoundError:
+            pass
+
     def gitify_hg(self, project_name, dirname, gitdir, spec):
         cert_options = []
         if not self.should_check_certificates(spec):
@@ -692,9 +710,7 @@ class Lorry(cliapp.Application):
             self.run_program(['hg', 'clone', '--quiet', *cert_options,
                               spec['url'], hgdir])
 
-        if not os.path.exists(gitdir):
-            self.needs_aggressive = True
-            self.run_program(['git', 'init', '--bare', gitdir])
+        self.needs_aggressive = self.ensure_gitdir(gitdir)
 
         # Since there are marks files in existing deployments that
         # have broken references, fix up the marks file before rather
@@ -737,8 +753,7 @@ class Lorry(cliapp.Application):
         else:
             self.progress('.. no need to run, nothing to do')
             return
-        if not os.path.exists(gitdir):
-            self.run_program(['git', 'init', '--bare', gitdir])
+        self.ensure_gitdir(gitdir)
 
         cmdline = ["%s.%s-importer" % (lorry_path, archive_type), archive_dest]
         self.run_program(cmdline, cwd=gitdir)
diff --git a/tests/cvs-single-commit.script b/tests/cvs-single-commit.script
index 728cc8b..a0639ba 100755
--- a/tests/cvs-single-commit.script
+++ b/tests/cvs-single-commit.script
@@ -2,7 +2,7 @@
 #
 # Tests converting a simple CVS repository to git.
 #
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012, 2020  Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,6 +23,14 @@ set -e
 logfile="$DATADIR/cvs-test-repo.log"
 workdir="$DATADIR/work-dir"
 
+normalize() {
+    sed -r -e '/hooks\/.*\.sample/d' \
+           -e "s/pack-[0-9a-z]+\.(idx|pack)$/pack-file/" \
+           -e "/\/objects\/info\/commit-graph$/d" \
+           -e "/\/objects\/pack\/pack-[0-9a-z]+\.bitmap$/d" \
+           -e "s|$DATADIR|DATADIR|g" "$@"
+}
+
 # CVS wants $USER, $LOGNAME, and $LOGNAME set to a real username.
 export USER=root
 export LOGNAME=$USER
@@ -44,3 +52,5 @@ export USERNAME=$USER
   echo "commit messages:"
   git log --pretty='%s' master
 )
+
+find "$workdir/cvs-test-repo" | LC_ALL=C sort | normalize
diff --git a/tests/cvs-single-commit.stdout b/tests/cvs-single-commit.stdout
index 91f58b0..6938eda 100644
--- a/tests/cvs-single-commit.stdout
+++ b/tests/cvs-single-commit.stdout
@@ -6,3 +6,24 @@ first line
 commit messages:
 first commit
 initial checkin
+DATADIR/work-dir/cvs-test-repo
+DATADIR/work-dir/cvs-test-repo/git-a
+DATADIR/work-dir/cvs-test-repo/git-a/HEAD
+DATADIR/work-dir/cvs-test-repo/git-a/branches
+DATADIR/work-dir/cvs-test-repo/git-a/config
+DATADIR/work-dir/cvs-test-repo/git-a/description
+DATADIR/work-dir/cvs-test-repo/git-a/hooks
+DATADIR/work-dir/cvs-test-repo/git-a/info
+DATADIR/work-dir/cvs-test-repo/git-a/info/exclude
+DATADIR/work-dir/cvs-test-repo/git-a/info/refs
+DATADIR/work-dir/cvs-test-repo/git-a/lorry-update-count
+DATADIR/work-dir/cvs-test-repo/git-a/objects
+DATADIR/work-dir/cvs-test-repo/git-a/objects/info
+DATADIR/work-dir/cvs-test-repo/git-a/objects/info/packs
+DATADIR/work-dir/cvs-test-repo/git-a/objects/pack
+DATADIR/work-dir/cvs-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/cvs-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/cvs-test-repo/git-a/packed-refs
+DATADIR/work-dir/cvs-test-repo/git-a/refs
+DATADIR/work-dir/cvs-test-repo/git-a/refs/heads
+DATADIR/work-dir/cvs-test-repo/git-a/refs/tags
diff --git a/tests/svn-single-commit.script b/tests/svn-single-commit.script
index 533f1dc..7ae2eb9 100755
--- a/tests/svn-single-commit.script
+++ b/tests/svn-single-commit.script
@@ -2,7 +2,7 @@
 #
 # Tests converting a simple SVN repository to git.
 #
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012, 2020  Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,6 +23,15 @@ set -e
 logfile="$DATADIR/svn-test-repo.log"
 workdir="$DATADIR/work-dir"
 
+normalize() {
+    sed -r -e '/hooks\/.*\.sample/d' \
+           -e "s/pack-[0-9a-z]+\.(idx|pack)$/pack-file/" \
+           -e "/\/objects\/info\/commit-graph$/d" \
+           -e "/\/objects\/pack\/pack-[0-9a-z]+\.bitmap$/d" \
+	   -e "s/\b[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}\b/UUID/" \
+           -e "s|$DATADIR|DATADIR|g" "$@"
+}
+
 "${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
   "$DATADIR/svn-test-repo.lorry" > /dev/null 2> /dev/null
 
@@ -39,3 +48,5 @@ workdir="$DATADIR/work-dir"
   # list the commit messages
   git log --pretty='%s' master
 )
+
+find "$workdir/svn-test-repo" | LC_ALL=C sort | normalize
diff --git a/tests/svn-single-commit.stdout b/tests/svn-single-commit.stdout
index eae50d1..8cb2926 100644
--- a/tests/svn-single-commit.stdout
+++ b/tests/svn-single-commit.stdout
@@ -1,3 +1,34 @@
 refs/heads/master
 first line
 first commit
+DATADIR/work-dir/svn-test-repo
+DATADIR/work-dir/svn-test-repo/git-a
+DATADIR/work-dir/svn-test-repo/git-a/HEAD
+DATADIR/work-dir/svn-test-repo/git-a/branches
+DATADIR/work-dir/svn-test-repo/git-a/config
+DATADIR/work-dir/svn-test-repo/git-a/description
+DATADIR/work-dir/svn-test-repo/git-a/hooks
+DATADIR/work-dir/svn-test-repo/git-a/info
+DATADIR/work-dir/svn-test-repo/git-a/info/exclude
+DATADIR/work-dir/svn-test-repo/git-a/info/refs
+DATADIR/work-dir/svn-test-repo/git-a/lorry-update-count
+DATADIR/work-dir/svn-test-repo/git-a/objects
+DATADIR/work-dir/svn-test-repo/git-a/objects/info
+DATADIR/work-dir/svn-test-repo/git-a/objects/info/packs
+DATADIR/work-dir/svn-test-repo/git-a/objects/pack
+DATADIR/work-dir/svn-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/svn-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/svn-test-repo/git-a/packed-refs
+DATADIR/work-dir/svn-test-repo/git-a/refs
+DATADIR/work-dir/svn-test-repo/git-a/refs/heads
+DATADIR/work-dir/svn-test-repo/git-a/refs/tags
+DATADIR/work-dir/svn-test-repo/git-a/svn
+DATADIR/work-dir/svn-test-repo/git-a/svn/.metadata
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/heads
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/heads/master
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/heads/master/.rev_map.UUID
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/heads/master/index
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/heads/master/unhandled.log
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/remotes
+DATADIR/work-dir/svn-test-repo/git-a/svn/refs/remotes/git-svn