Merge branch 'bwh/crash-proofing' into 'master'

lorry: Crash-proof updates to working git repos with A/B switching

Closes #8

See merge request CodethinkLabs/lorry/lorry!14

23 files changed, 478 insertions, 139 deletions

diff --git a/lorry b/lorry
index b42861b..65f1cb3 100755
--- a/lorry
+++ b/lorry
@@ -41,6 +41,9 @@ __version__ = '0.0'
 lorry_path = os.path.realpath(__file__)
 
 
+UPDATE_COUNT_NAME = 'lorry-update-count'
+
+
 def file_missing_or_empty(filename):
     ''' A more comprehensive alternative to os.path.exists(). '''
     return (not os.path.isfile(filename)) or (os.path.getsize(filename) <= 0)
@@ -343,15 +346,12 @@ class Lorry(cliapp.Application):
         dirname = self.dirname(name)
         if not os.path.exists(dirname):
             os.mkdir(dirname)
-        gitdir = os.path.join(dirname, 'git')
-
-        time = datetime.now().strftime('%F-%T')
 
-        pre_update_name = 'git-pre-update'
-        pre_update_backup_suffix = (pre_update_name + '-' + time
-            if self.settings['keep-multiple-backups'] else pre_update_name)
+        self.migrate_oldstyle_repos(dirname)
+        temp_repo, active_repo, next_update_count = \
+            self.prepare_working_repos(dirname)
 
-        pre_update_backup_dir = os.path.join(dirname, pre_update_backup_suffix)
+        time = datetime.now().strftime('%F-%T')
 
         post_fail_name = 'git-post-fail'
         post_fail_backup_suffix = (post_fail_name + '-' + time
@@ -360,79 +360,130 @@ class Lorry(cliapp.Application):
         post_fail_backup_dir = os.path.join(dirname, post_fail_backup_suffix)
 
         if not self.settings['keep-multiple-backups']:
-            # remove previous backups if they exist
-            if os.path.exists(pre_update_backup_dir):
-                shutil.rmtree(pre_update_backup_dir)
-
             if os.path.exists(post_fail_backup_dir):
                 shutil.rmtree(post_fail_backup_dir)
 
-        backupdir = self.backup_gitdir(name, gitdir, pre_update_backup_dir)
-
         try:
             self.needs_aggressive = False
-            table[vcstype](name, dirname, gitdir, spec)
+            table[vcstype](name, dirname, temp_repo, spec)
             if self.settings['repack']:
                 self.progress('.. repacking %s git repository' % name)
                 self.run_program(['git', 'config',
-                                  'pack.windowMemory', '128M'], cwd=gitdir)
+                                  'pack.windowMemory', '128M'], cwd=temp_repo)
                 args = ['git', 'gc']
                 if self.needs_aggressive:
                     args += ['--aggressive']
-                self.run_program(args, cwd=gitdir)
+                self.run_program(args, cwd=temp_repo)
 
-            self.bundle(name, gitdir)
-            self.make_tarball(name, gitdir)
+            self.bundle(name, temp_repo)
+            self.make_tarball(name, temp_repo)
+
+            self.write_update_count(temp_repo, next_update_count)
+            active_repo = temp_repo
         except:
-            if backupdir is not None:
-                faildir = self.backup_gitdir(name, gitdir, post_fail_backup_dir)
-                self.restore_backup(name, backupdir, gitdir)
+            if active_repo is not None:
+                os.rename(temp_repo, post_fail_backup_dir)
                 self.output.write('Mirror of %s failed, state before mirror '
                                   'is saved at %s and state after mirror is '
-                                  'saved at %s\n' % (name, backupdir, faildir))
+                                  'saved at %s\n'
+                                  % (name, active_repo, post_fail_backup_dir))
                 logging.debug('Mirror of %s failed, state before mirror '
                               'is saved at %s and state after mirror is '
-                              'saved at %s\n', name, backupdir, faildir)
+                              'saved at %s\n',
+                              name, active_repo, post_fail_backup_dir)
             raise
 
         if not self.settings['pull-only']:
             if len(self.settings['mirror-base-url-push']) > 0:
                 if 'refspecs' in spec:
-                    self.push_to_mirror_server(name, gitdir, spec['refspecs'])
+                    self.push_to_mirror_server(name, active_repo, spec['refspecs'])
                 else:
-                    self.push_to_mirror_server(name, gitdir)
-
-        if backupdir is not None:
-            self.progress('.. removing %s git repository backup' % name)
-            shutil.rmtree(backupdir)
-
-    def restore_backup(self, name, backupdir, gitdir):
-        self.progress('.. restoring %s good git repository' % name)
-        dotgit = os.path.join(gitdir, '.git')
-        if not os.path.exists(dotgit):
-            dotgit = gitdir
-        shutil.rmtree(dotgit)
-        self.copy_gitdir(backupdir, dotgit)
+                    self.push_to_mirror_server(name, active_repo)
+
+    def migrate_oldstyle_repos(self, dirname):
+        # Migrate old-style active repository
+        old_repo = os.path.join(dirname, 'git')
+        if os.path.exists(old_repo):
+            new_repo = os.path.join(dirname, 'git-a')
+            if os.path.exists(new_repo):
+                msg = ('Found both old %s and new %s directories; '
+                       'not migrating\n' % (old_repo, new_repo))
+                self.output.write(msg)
+                logging.warning(msg)
+            else:
+                # If it has a .git subdirectory, use that
+                old_gitdir = os.path.join(old_repo, '.git')
+                if not os.path.exists(old_gitdir):
+                    old_gitdir = old_repo
+
+                self.write_update_count(old_gitdir, 1)
+
+                # Move it to new name, and remove top-level directory if we
+                # moved the .git subdirectory
+                os.rename(old_gitdir, new_repo)
+                if old_repo != old_gitdir:
+                    shutil.rmtree(old_repo)
+
+        # Remove old-style backup repository
+        old_repo = os.path.join(dirname, 'git-pre-update')
+        if os.path.exists(old_repo):
+            shutil.rmtree(old_repo)
+
+    def prepare_working_repos(self, dirname):
+        # Determine which repository is active (has highest update
+        # count) and which we will create or replace
+        repos = []
+        for repo in [os.path.join(dirname, 'git-a'),
+                     os.path.join(dirname, 'git-b')]:
+            tstamp = -1
+            count = 0
+            try:
+                count_name = os.path.join(repo, UPDATE_COUNT_NAME)
+                with open(count_name, 'r') as count_file:
+                    tstamp = os.stat(count_file.fileno()).st_mtime
+                    count = int(count_file.readline())
+            except (FileNotFoundError, ValueError):
+                pass
+            repos.append((count, tstamp, repo))
+        repos.sort()
+        temp_count, _, temp_repo = repos[0]
+        active_count, active_tstamp, active_repo = repos[1]
+
+        # Remove/rename temporary repository
+        if os.path.exists(temp_repo):
+            # If this was the result of a successful conversion, and
+            # multiple backups are enabled, rename it.  We name it
+            # using the timestamp of the active repository, i.e. the
+            # time that this repository became inactive.
+            if temp_count > 0 and self.settings['keep-multiple-backups']:
+                time = datetime.fromtimestamp(active_tstamp) \
+                               .strftime('%F-%T')
+                os.rename(temp_repo,
+                          os.path.join(dirname, 'git-pre-update-' + time))
+            else:
+                shutil.rmtree(temp_repo)
 
-    def backup_gitdir(self, name, gitdir, backupdir):
-        dotgit = os.path.join(gitdir, '.git')
-        if not os.path.exists(dotgit):
-            dotgit = gitdir
+        if active_count == 0:
+            return temp_repo, None, 1
 
-        self.progress('.. backing up %s git repository to %s' % (name, backupdir))
+        self.copy_gitdir(active_repo, temp_repo)
+        return temp_repo, active_repo, active_count + 1
 
-        return self.copy_gitdir(dotgit, backupdir)
+    def write_update_count(self, gitdir, count):
+        count_name = os.path.join(gitdir, UPDATE_COUNT_NAME)
+        with open(count_name, 'w') as count_file:
+            count_file.write('%d\n' % count)
 
     def copy_gitdir(self, source, dest):
         if not os.path.exists(source):
             return None
 
-        # copy everything except the objects dir
-        def ignoreobjects(dirname, filenames):
+        # copy everything except the objects dir and update count
+        def ignore_filter(dirname, filenames):
             if dirname.endswith(source):
-                return ['objects']
+                return ['objects', UPDATE_COUNT_NAME]
             return []
-        shutil.copytree(source, dest, ignore=ignoreobjects)
+        shutil.copytree(source, dest, ignore=ignore_filter)
 
         # hardlink the objects
         sourceobjects = os.path.join(source, 'objects')
diff --git a/tests/bzr-single-commit.script b/tests/bzr-single-commit.script
index 9b38552..80b9d99 100755
--- a/tests/bzr-single-commit.script
+++ b/tests/bzr-single-commit.script
@@ -29,7 +29,7 @@ workdir="$DATADIR/work-dir"
 
 # verify that the git repository was set up correctly
 (
-  cd "$workdir/bzr-test-repo/git/"
+  cd "$workdir/bzr-test-repo/git-a/"
 
   # list the branches
   git show-ref | cut -d' ' -f2
diff --git a/tests/cvs-single-commit.script b/tests/cvs-single-commit.script
index 4ce2587..728cc8b 100755
--- a/tests/cvs-single-commit.script
+++ b/tests/cvs-single-commit.script
@@ -33,7 +33,7 @@ export USERNAME=$USER
 
 # verify that the git repository was created successfully
 (
-  cd "$workdir/cvs-test-repo/git/"
+  cd "$workdir/cvs-test-repo/git-a/"
 
   echo "branches:"
   git show-ref | cut -d' ' -f2 | LC_ALL=C sort
diff --git a/tests/extended-headers-ignored.script b/tests/extended-headers-ignored.script
index 6f49ae4..dcc01fc 100755
--- a/tests/extended-headers-ignored.script
+++ b/tests/extended-headers-ignored.script
@@ -34,6 +34,6 @@ workdir="$DATADIR/work-dir"
   "$DATADIR/extended-headers-ignored-test.lorry" > /dev/null 2> /dev/null
 
 repodst="$DATADIR/extended-headers"
-git clone "$workdir/extended-headers-test-repo/git" "$repodst" >/dev/null 2>&1
+git clone "$workdir/extended-headers-test-repo/git-a" "$repodst" >/dev/null 2>&1
 
 ls "$repodst"
diff --git a/tests/git-backup-on-error-multiple-backups.script b/tests/git-backup-on-error-multiple-backups.script
index a7e5148..763ff3f 100755
--- a/tests/git-backup-on-error-multiple-backups.script
+++ b/tests/git-backup-on-error-multiple-backups.script
@@ -51,4 +51,5 @@ then
     exit 1
 else
     find "$workdir/git-backup-test-repo" | LC_ALL=C sort | normalize
+    grep -H . "$workdir/git-backup-test-repo/git"*"/lorry-update-count" | normalize
 fi
diff --git a/tests/git-backup-on-error-multiple-backups.stdout b/tests/git-backup-on-error-multiple-backups.stdout
index d918599..5b1709c 100644
--- a/tests/git-backup-on-error-multiple-backups.stdout
+++ b/tests/git-backup-on-error-multiple-backups.stdout
@@ -1,6 +1,26 @@
-Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME
+Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-a and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME
 DATADIR/work-dir/git-backup-test-repo
-DATADIR/work-dir/git-backup-test-repo/git
+DATADIR/work-dir/git-backup-test-repo/git-a
+DATADIR/work-dir/git-backup-test-repo/git-a/FETCH_HEAD
+DATADIR/work-dir/git-backup-test-repo/git-a/HEAD
+DATADIR/work-dir/git-backup-test-repo/git-a/branches
+DATADIR/work-dir/git-backup-test-repo/git-a/config
+DATADIR/work-dir/git-backup-test-repo/git-a/description
+DATADIR/work-dir/git-backup-test-repo/git-a/hooks
+DATADIR/work-dir/git-backup-test-repo/git-a/info
+DATADIR/work-dir/git-backup-test-repo/git-a/info/exclude
+DATADIR/work-dir/git-backup-test-repo/git-a/info/refs
+DATADIR/work-dir/git-backup-test-repo/git-a/lorry-update-count
+DATADIR/work-dir/git-backup-test-repo/git-a/objects
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/info
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/info/packs
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/git-backup-test-repo/git-a/packed-refs
+DATADIR/work-dir/git-backup-test-repo/git-a/refs
+DATADIR/work-dir/git-backup-test-repo/git-a/refs/heads
+DATADIR/work-dir/git-backup-test-repo/git-a/refs/tags
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/FETCH_HEAD
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/HEAD
@@ -21,42 +41,4 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/packed-refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads
 DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/tags
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/FETCH_HEAD
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/HEAD
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/branches
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/config
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/description
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/exclude
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/refs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/info
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/info/packs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/pack
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/packed-refs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads
-DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/tags
-DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD
-DATADIR/work-dir/git-backup-test-repo/git/HEAD
-DATADIR/work-dir/git-backup-test-repo/git/branches
-DATADIR/work-dir/git-backup-test-repo/git/config
-DATADIR/work-dir/git-backup-test-repo/git/description
-DATADIR/work-dir/git-backup-test-repo/git/hooks
-DATADIR/work-dir/git-backup-test-repo/git/info
-DATADIR/work-dir/git-backup-test-repo/git/info/exclude
-DATADIR/work-dir/git-backup-test-repo/git/info/refs
-DATADIR/work-dir/git-backup-test-repo/git/objects
-DATADIR/work-dir/git-backup-test-repo/git/objects/info
-DATADIR/work-dir/git-backup-test-repo/git/objects/info/packs
-DATADIR/work-dir/git-backup-test-repo/git/objects/pack
-DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git/packed-refs
-DATADIR/work-dir/git-backup-test-repo/git/refs
-DATADIR/work-dir/git-backup-test-repo/git/refs/heads
-DATADIR/work-dir/git-backup-test-repo/git/refs/tags
+DATADIR/work-dir/git-backup-test-repo/git-a/lorry-update-count:1
diff --git a/tests/git-backup-on-error-single-backup.script b/tests/git-backup-on-error-single-backup.script
index b69f612..f66103a 100755
--- a/tests/git-backup-on-error-single-backup.script
+++ b/tests/git-backup-on-error-single-backup.script
@@ -50,4 +50,5 @@ then
     exit 1
 else
     find "$workdir/git-backup-test-repo" | LC_ALL=C sort | normalize
+    grep -H . "$workdir/git-backup-test-repo/git"*"/lorry-update-count" | normalize
 fi
diff --git a/tests/git-backup-on-error-single-backup.stdout b/tests/git-backup-on-error-single-backup.stdout
index db622ef..1c56b6a 100644
--- a/tests/git-backup-on-error-single-backup.stdout
+++ b/tests/git-backup-on-error-single-backup.stdout
@@ -1,6 +1,26 @@
-Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-pre-update and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail
+Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-a and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail
 DATADIR/work-dir/git-backup-test-repo
-DATADIR/work-dir/git-backup-test-repo/git
+DATADIR/work-dir/git-backup-test-repo/git-a
+DATADIR/work-dir/git-backup-test-repo/git-a/FETCH_HEAD
+DATADIR/work-dir/git-backup-test-repo/git-a/HEAD
+DATADIR/work-dir/git-backup-test-repo/git-a/branches
+DATADIR/work-dir/git-backup-test-repo/git-a/config
+DATADIR/work-dir/git-backup-test-repo/git-a/description
+DATADIR/work-dir/git-backup-test-repo/git-a/hooks
+DATADIR/work-dir/git-backup-test-repo/git-a/info
+DATADIR/work-dir/git-backup-test-repo/git-a/info/exclude
+DATADIR/work-dir/git-backup-test-repo/git-a/info/refs
+DATADIR/work-dir/git-backup-test-repo/git-a/lorry-update-count
+DATADIR/work-dir/git-backup-test-repo/git-a/objects
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/info
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/info/packs
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/git-backup-test-repo/git-a/packed-refs
+DATADIR/work-dir/git-backup-test-repo/git-a/refs
+DATADIR/work-dir/git-backup-test-repo/git-a/refs/heads
+DATADIR/work-dir/git-backup-test-repo/git-a/refs/tags
 DATADIR/work-dir/git-backup-test-repo/git-post-fail
 DATADIR/work-dir/git-backup-test-repo/git-post-fail/FETCH_HEAD
 DATADIR/work-dir/git-backup-test-repo/git-post-fail/HEAD
@@ -21,42 +41,4 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail/packed-refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail/refs
 DATADIR/work-dir/git-backup-test-repo/git-post-fail/refs/heads
 DATADIR/work-dir/git-backup-test-repo/git-post-fail/refs/tags
-DATADIR/work-dir/git-backup-test-repo/git-pre-update
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/FETCH_HEAD
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/HEAD
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/branches
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/config
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/description
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/hooks
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/info
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/info/exclude
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/info/refs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/info
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/info/packs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/pack
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/packed-refs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/refs
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/refs/heads
-DATADIR/work-dir/git-backup-test-repo/git-pre-update/refs/tags
-DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD
-DATADIR/work-dir/git-backup-test-repo/git/HEAD
-DATADIR/work-dir/git-backup-test-repo/git/branches
-DATADIR/work-dir/git-backup-test-repo/git/config
-DATADIR/work-dir/git-backup-test-repo/git/description
-DATADIR/work-dir/git-backup-test-repo/git/hooks
-DATADIR/work-dir/git-backup-test-repo/git/info
-DATADIR/work-dir/git-backup-test-repo/git/info/exclude
-DATADIR/work-dir/git-backup-test-repo/git/info/refs
-DATADIR/work-dir/git-backup-test-repo/git/objects
-DATADIR/work-dir/git-backup-test-repo/git/objects/info
-DATADIR/work-dir/git-backup-test-repo/git/objects/info/packs
-DATADIR/work-dir/git-backup-test-repo/git/objects/pack
-DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file
-DATADIR/work-dir/git-backup-test-repo/git/packed-refs
-DATADIR/work-dir/git-backup-test-repo/git/refs
-DATADIR/work-dir/git-backup-test-repo/git/refs/heads
-DATADIR/work-dir/git-backup-test-repo/git/refs/tags
+DATADIR/work-dir/git-backup-test-repo/git-a/lorry-update-count:1
diff --git a/tests/git-single-commit.script b/tests/git-single-commit.script
index 7a980b2..9c4813e 100755
--- a/tests/git-single-commit.script
+++ b/tests/git-single-commit.script
@@ -29,7 +29,7 @@ workdir="$DATADIR/work-dir"
 
 # verify that the git repository was set up correctly
 (
-  cd "$workdir/git-test-repo/git/"
+  cd "$workdir/git-test-repo/git-a/"
 
   # list the branches
   git show-ref | cut -d' ' -f2
diff --git a/tests/hg-incremental.script b/tests/hg-incremental.script
index 16ca350..489efa7 100755
--- a/tests/hg-incremental.script
+++ b/tests/hg-incremental.script
@@ -40,7 +40,7 @@ workdir="$DATADIR/work-dir"
 
 # verify that the git repository was created correctly
 (
-  cd "$workdir/hg-test-repo/git/"
+  cd "$workdir/hg-test-repo/git-b/"
 
   # list the branches
   git show-ref | cut -d' ' -f2
diff --git a/tests/hg-single-commit.script b/tests/hg-single-commit.script
index e9814b9..0b95dc9 100755
--- a/tests/hg-single-commit.script
+++ b/tests/hg-single-commit.script
@@ -29,7 +29,7 @@ workdir="$DATADIR/work-dir"
 
 # verify that the git repository was created correctly
 (
-  cd "$workdir/hg-test-repo/git/"
+  cd "$workdir/hg-test-repo/git-a/"
 
   # list the branches
   git show-ref | cut -d' ' -f2
diff --git a/tests/hg-unnamed-head.script b/tests/hg-unnamed-head.script
index 512938f..1811645 100755
--- a/tests/hg-unnamed-head.script
+++ b/tests/hg-unnamed-head.script
@@ -30,7 +30,7 @@ grep -o 'pruned [0-9]* unreachable commit' < "$logfile" || true
 
 # verify that the git repository was created correctly
 (
-  cd "$workdir/hg-test-repo/git/"
+  cd "$workdir/hg-test-repo/git-a/"
 
   # check number of commits generated
   wc -l hg2git-marks
diff --git a/tests/make-tarball.script b/tests/make-tarball.script
index 2576447..46618cd 100755
--- a/tests/make-tarball.script
+++ b/tests/make-tarball.script
@@ -29,4 +29,4 @@ workdir="$DATADIR/work-dir"
 
 # verify that we can see the tarball generated of the git tree
 
-test -r "${workdir}/make-tarball-repo-bzip2/git/"*"make_tarball_repo_bzip2.tar"
+test -r "${workdir}/make-tarball-repo-bzip2/git-a/"*"make_tarball_repo_bzip2.tar"
diff --git a/tests/migrate-oldstyle-repos.script b/tests/migrate-oldstyle-repos.script
new file mode 100755
index 0000000..c6be1be
--- /dev/null
+++ b/tests/migrate-oldstyle-repos.script
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Tests migration of old-style working repositories
+#
+# Copyright (C) 2012-2013, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+set -o pipefail
+
+repo_name="migrate-test-repo"
+logfile="$DATADIR/$repo_name.log"
+workdir="$DATADIR/work-dir"
+repo="$DATADIR/$repo_name"
+
+normalize() {
+    sed -r -e '/hooks\/.*\.sample/d' \
+           -e "s/pack-[0-9a-z]+\.(idx|pack)$/pack-file/" \
+           -e "/\/objects\/info\/commit-graph$/d" \
+           -e "/\/objects\/pack\/pack-[0-9a-z]+\.bitmap$/d" \
+           -e "s|$DATADIR|DATADIR|g" "$@"
+}
+
+# update it
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \
+  "$DATADIR/migrate-test-repo.lorry"
+
+find "$workdir/migrate-test-repo" | LC_ALL=C sort | normalize
+grep -H . "$workdir/migrate-test-repo/git"*"/lorry-update-count" | normalize
diff --git a/tests/migrate-oldstyle-repos.setup b/tests/migrate-oldstyle-repos.setup
new file mode 100755
index 0000000..eea87eb
--- /dev/null
+++ b/tests/migrate-oldstyle-repos.setup
@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# Tests migration of old-style working repositories
+#
+# Copyright (C) 2012-2013, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+# create the repository
+repo="$DATADIR/migrate-test-repo"
+mkdir "$repo"
+cd "$repo"
+git init --quiet
+
+# add the test file
+echo "first line" > test.txt
+git add test.txt
+
+# make a commit
+git commit --quiet -m "first commit"
+
+# create the .lorry file for the repository
+cat <<EOF > $DATADIR/migrate-test-repo.lorry
+{
+  "migrate-test-repo": {
+    "type": "git",
+    "url": "file://$repo"
+  }
+}
+EOF
+
+# create the working directory
+workdir="$DATADIR/work-dir"
+test -d "$workdir" || mkdir "$workdir"
+
+# create old-style working repositories
+for work_repo in "$workdir/migrate-test-repo/git" \
+                 "$workdir/migrate-test-repo/git-pre-update"; do
+    git init --bare "$work_repo"
+    git -C "$work_repo" fetch "$repo" '+refs/heads/*:refs/heads/*'
+    git -C "$work_repo" gc --aggressive
+done
diff --git a/tests/migrate-oldstyle-repos.stdout b/tests/migrate-oldstyle-repos.stdout
new file mode 100644
index 0000000..9402534
--- /dev/null
+++ b/tests/migrate-oldstyle-repos.stdout
@@ -0,0 +1,45 @@
+DATADIR/work-dir/migrate-test-repo
+DATADIR/work-dir/migrate-test-repo/git-a
+DATADIR/work-dir/migrate-test-repo/git-a/FETCH_HEAD
+DATADIR/work-dir/migrate-test-repo/git-a/HEAD
+DATADIR/work-dir/migrate-test-repo/git-a/branches
+DATADIR/work-dir/migrate-test-repo/git-a/config
+DATADIR/work-dir/migrate-test-repo/git-a/description
+DATADIR/work-dir/migrate-test-repo/git-a/hooks
+DATADIR/work-dir/migrate-test-repo/git-a/info
+DATADIR/work-dir/migrate-test-repo/git-a/info/exclude
+DATADIR/work-dir/migrate-test-repo/git-a/info/refs
+DATADIR/work-dir/migrate-test-repo/git-a/lorry-update-count
+DATADIR/work-dir/migrate-test-repo/git-a/objects
+DATADIR/work-dir/migrate-test-repo/git-a/objects/info
+DATADIR/work-dir/migrate-test-repo/git-a/objects/info/packs
+DATADIR/work-dir/migrate-test-repo/git-a/objects/pack
+DATADIR/work-dir/migrate-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/migrate-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/migrate-test-repo/git-a/packed-refs
+DATADIR/work-dir/migrate-test-repo/git-a/refs
+DATADIR/work-dir/migrate-test-repo/git-a/refs/heads
+DATADIR/work-dir/migrate-test-repo/git-a/refs/tags
+DATADIR/work-dir/migrate-test-repo/git-b
+DATADIR/work-dir/migrate-test-repo/git-b/FETCH_HEAD
+DATADIR/work-dir/migrate-test-repo/git-b/HEAD
+DATADIR/work-dir/migrate-test-repo/git-b/branches
+DATADIR/work-dir/migrate-test-repo/git-b/config
+DATADIR/work-dir/migrate-test-repo/git-b/description
+DATADIR/work-dir/migrate-test-repo/git-b/hooks
+DATADIR/work-dir/migrate-test-repo/git-b/info
+DATADIR/work-dir/migrate-test-repo/git-b/info/exclude
+DATADIR/work-dir/migrate-test-repo/git-b/info/refs
+DATADIR/work-dir/migrate-test-repo/git-b/lorry-update-count
+DATADIR/work-dir/migrate-test-repo/git-b/objects
+DATADIR/work-dir/migrate-test-repo/git-b/objects/info
+DATADIR/work-dir/migrate-test-repo/git-b/objects/info/packs
+DATADIR/work-dir/migrate-test-repo/git-b/objects/pack
+DATADIR/work-dir/migrate-test-repo/git-b/objects/pack/pack-file
+DATADIR/work-dir/migrate-test-repo/git-b/objects/pack/pack-file
+DATADIR/work-dir/migrate-test-repo/git-b/packed-refs
+DATADIR/work-dir/migrate-test-repo/git-b/refs
+DATADIR/work-dir/migrate-test-repo/git-b/refs/heads
+DATADIR/work-dir/migrate-test-repo/git-b/refs/tags
+DATADIR/work-dir/migrate-test-repo/git-a/lorry-update-count:1
+DATADIR/work-dir/migrate-test-repo/git-b/lorry-update-count:2
diff --git a/tests/migrate-oldstyle-repos.teardown b/tests/migrate-oldstyle-repos.teardown
new file mode 100755
index 0000000..19d1cb1
--- /dev/null
+++ b/tests/migrate-oldstyle-repos.teardown
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Tests migration of old-style working repositories
+#
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+rm -r "$DATADIR/migrate-test-repo.log" "$DATADIR/work-dir"
diff --git a/tests/repo-update-count.script b/tests/repo-update-count.script
new file mode 100755
index 0000000..db68b12
--- /dev/null
+++ b/tests/repo-update-count.script
@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Tests update counts in working repositories
+#
+# Copyright (C) 2012-2013, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+set -o pipefail
+
+repo_name="update-count-test-repo"
+logfile="$DATADIR/$repo_name.log"
+workdir="$DATADIR/work-dir"
+repo="$DATADIR/$repo_name"
+
+normalize() {
+    sed -r -e '/hooks\/.*\.sample/d' \
+           -e "s/pack-[0-9a-z]+\.(idx|pack)$/pack-file/" \
+           -e "/\/objects\/info\/commit-graph$/d" \
+           -e "/\/objects\/pack\/pack-[0-9a-z]+\.bitmap$/d" \
+           -e "s|$DATADIR|DATADIR|g" "$@"
+}
+
+# update it 3 times, which should write a=1, b=2, a=3
+for i in 1 2 3; do
+    "${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \
+      "$DATADIR/update-count-test-repo.lorry"
+done
+
+find "$workdir/update-count-test-repo" | LC_ALL=C sort | normalize
+grep -H . "$workdir/update-count-test-repo/git"*"/lorry-update-count" | normalize
diff --git a/tests/repo-update-count.setup b/tests/repo-update-count.setup
new file mode 100755
index 0000000..3df1c39
--- /dev/null
+++ b/tests/repo-update-count.setup
@@ -0,0 +1,48 @@
+#!/bin/sh
+#
+# Tests update counts in working repositories
+#
+# Copyright (C) 2012-2013, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+# create the repository
+repo="$DATADIR/update-count-test-repo"
+mkdir "$repo"
+cd "$repo"
+git init --quiet
+
+# add the test file
+echo "first line" > test.txt
+git add test.txt
+
+# make a commit
+git commit --quiet -m "first commit"
+
+# create the .lorry file for the repository
+cat <<EOF > $DATADIR/update-count-test-repo.lorry
+{
+  "update-count-test-repo": {
+    "type": "git",
+    "url": "file://$repo"
+  }
+}
+EOF
+
+# create the working directory
+workdir="$DATADIR/work-dir"
+test -d "$workdir" || mkdir "$workdir"
diff --git a/tests/repo-update-count.stdout b/tests/repo-update-count.stdout
new file mode 100644
index 0000000..6edd8c0
--- /dev/null
+++ b/tests/repo-update-count.stdout
@@ -0,0 +1,45 @@
+DATADIR/work-dir/update-count-test-repo
+DATADIR/work-dir/update-count-test-repo/git-a
+DATADIR/work-dir/update-count-test-repo/git-a/FETCH_HEAD
+DATADIR/work-dir/update-count-test-repo/git-a/HEAD
+DATADIR/work-dir/update-count-test-repo/git-a/branches
+DATADIR/work-dir/update-count-test-repo/git-a/config
+DATADIR/work-dir/update-count-test-repo/git-a/description
+DATADIR/work-dir/update-count-test-repo/git-a/hooks
+DATADIR/work-dir/update-count-test-repo/git-a/info
+DATADIR/work-dir/update-count-test-repo/git-a/info/exclude
+DATADIR/work-dir/update-count-test-repo/git-a/info/refs
+DATADIR/work-dir/update-count-test-repo/git-a/lorry-update-count
+DATADIR/work-dir/update-count-test-repo/git-a/objects
+DATADIR/work-dir/update-count-test-repo/git-a/objects/info
+DATADIR/work-dir/update-count-test-repo/git-a/objects/info/packs
+DATADIR/work-dir/update-count-test-repo/git-a/objects/pack
+DATADIR/work-dir/update-count-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/update-count-test-repo/git-a/objects/pack/pack-file
+DATADIR/work-dir/update-count-test-repo/git-a/packed-refs
+DATADIR/work-dir/update-count-test-repo/git-a/refs
+DATADIR/work-dir/update-count-test-repo/git-a/refs/heads
+DATADIR/work-dir/update-count-test-repo/git-a/refs/tags
+DATADIR/work-dir/update-count-test-repo/git-b
+DATADIR/work-dir/update-count-test-repo/git-b/FETCH_HEAD
+DATADIR/work-dir/update-count-test-repo/git-b/HEAD
+DATADIR/work-dir/update-count-test-repo/git-b/branches
+DATADIR/work-dir/update-count-test-repo/git-b/config
+DATADIR/work-dir/update-count-test-repo/git-b/description
+DATADIR/work-dir/update-count-test-repo/git-b/hooks
+DATADIR/work-dir/update-count-test-repo/git-b/info
+DATADIR/work-dir/update-count-test-repo/git-b/info/exclude
+DATADIR/work-dir/update-count-test-repo/git-b/info/refs
+DATADIR/work-dir/update-count-test-repo/git-b/lorry-update-count
+DATADIR/work-dir/update-count-test-repo/git-b/objects
+DATADIR/work-dir/update-count-test-repo/git-b/objects/info
+DATADIR/work-dir/update-count-test-repo/git-b/objects/info/packs
+DATADIR/work-dir/update-count-test-repo/git-b/objects/pack
+DATADIR/work-dir/update-count-test-repo/git-b/objects/pack/pack-file
+DATADIR/work-dir/update-count-test-repo/git-b/objects/pack/pack-file
+DATADIR/work-dir/update-count-test-repo/git-b/packed-refs
+DATADIR/work-dir/update-count-test-repo/git-b/refs
+DATADIR/work-dir/update-count-test-repo/git-b/refs/heads
+DATADIR/work-dir/update-count-test-repo/git-b/refs/tags
+DATADIR/work-dir/update-count-test-repo/git-a/lorry-update-count:3
+DATADIR/work-dir/update-count-test-repo/git-b/lorry-update-count:2
diff --git a/tests/repo-update-count.teardown b/tests/repo-update-count.teardown
new file mode 100755
index 0000000..b400abf
--- /dev/null
+++ b/tests/repo-update-count.teardown
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Tests update counts in working repositories
+#
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+rm -r "$DATADIR/update-count-test-repo.log" "$DATADIR/work-dir"
diff --git a/tests/svn-single-commit.script b/tests/svn-single-commit.script
index 32a9cca..533f1dc 100755
--- a/tests/svn-single-commit.script
+++ b/tests/svn-single-commit.script
@@ -28,7 +28,7 @@ workdir="$DATADIR/work-dir"
 
 # verify that the git repository was created successfully
 (
-  cd "$workdir/svn-test-repo/git/"
+  cd "$workdir/svn-test-repo/git-a/"
 
   # list the branches
   git show-ref | cut -d' ' -f2
diff --git a/tests/tar-single-commit.script b/tests/tar-single-commit.script
index 1562446..4d91417 100755
--- a/tests/tar-single-commit.script
+++ b/tests/tar-single-commit.script
@@ -28,7 +28,7 @@ workdir="$DATADIR/work-dir"
 
 # verify that the git repositories were created successfully
 for FORMAT in "gzip" "bzip2" "lzma"; do
-  cd "$workdir/tar-test-repo-$FORMAT/git/"
+  cd "$workdir/tar-test-repo-$FORMAT/git-a/"
 
   echo "$FORMAT"