From 50383f3bcce75d897760bb2c606ffea865f352da Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Thu, 6 Aug 2020 16:37:41 +0100 Subject: lorry: Crash-proof updates to working git repos with A/B switching lorry currently updates a git repository in its working area like this: 1. Delete old backup and debug repositories 2. Copy active repository (git) to backup (git-pre-update) 3. Perform conversion in active repository 4. If conversion failed, restore the backup: 1. Copy active repository to debug (git-post-fail) 2. Delete active repository 3. Copy backup repository to active 5. If conversion succeeded: delete git-pre-update If step 3 or 4 is interrupted, for example because of a system reboot, the active repository may be left missing or broken. On the next update, the backup will be deleted. Replace this with the following process: 1. Look for repositories 'git-a' and 'git-b'. The one with a higher update count (or the only one that exists and has an update count file) is the active repository. The other is the the temporary repository. In case neither exists, 'git-a' is the temporary repository. 2. Delete old temporary repository, if it exists 3. Copy active repository, if it exists, to temporary repository (excluding the update count file) 4. Perform conversion in temporary repository 5. If conversion succeeded, make the temporary repository active by writing the new update count file 6. If conversion failed, move the temporary repository to debug as before We need to migrate the working state, so if there is a 'git' and no 'git-a', add an update count of 1 to 'git' and move it to 'git-a'. Also delete the old-style backup repository. Update test cases to expect the new filenames. lorry should now be crash-proof, in the sense that killing it at any point will leave the last successful update in place. However, an unclean system shutdown (e.g. kernel crash) that does not flush all writes to persistent storage may still result in the active repository being left broken. Closes: #8. --- tests/bzr-single-commit.script | 2 +- tests/cvs-single-commit.script | 2 +- tests/extended-headers-ignored.script | 2 +- tests/git-backup-on-error-multiple-backups.stdout | 63 ++++++++--------------- tests/git-backup-on-error-single-backup.stdout | 63 ++++++++--------------- tests/git-single-commit.script | 2 +- tests/hg-incremental.script | 2 +- tests/hg-single-commit.script | 2 +- tests/hg-unnamed-head.script | 2 +- tests/make-tarball.script | 2 +- tests/svn-single-commit.script | 2 +- tests/tar-single-commit.script | 2 +- 12 files changed, 54 insertions(+), 92 deletions(-) (limited to 'tests') diff --git a/tests/bzr-single-commit.script b/tests/bzr-single-commit.script index 9b38552..80b9d99 100755 --- a/tests/bzr-single-commit.script +++ b/tests/bzr-single-commit.script @@ -29,7 +29,7 @@ workdir="$DATADIR/work-dir" # verify that the git repository was set up correctly ( - cd "$workdir/bzr-test-repo/git/" + cd "$workdir/bzr-test-repo/git-a/" # list the branches git show-ref | cut -d' ' -f2 diff --git a/tests/cvs-single-commit.script b/tests/cvs-single-commit.script index 4ce2587..728cc8b 100755 --- a/tests/cvs-single-commit.script +++ b/tests/cvs-single-commit.script @@ -33,7 +33,7 @@ export USERNAME=$USER # verify that the git repository was created successfully ( - cd "$workdir/cvs-test-repo/git/" + cd "$workdir/cvs-test-repo/git-a/" echo "branches:" git show-ref | cut -d' ' -f2 | LC_ALL=C sort diff --git a/tests/extended-headers-ignored.script b/tests/extended-headers-ignored.script index 6f49ae4..dcc01fc 100755 --- a/tests/extended-headers-ignored.script +++ b/tests/extended-headers-ignored.script @@ -34,6 +34,6 @@ workdir="$DATADIR/work-dir" "$DATADIR/extended-headers-ignored-test.lorry" > /dev/null 2> /dev/null repodst="$DATADIR/extended-headers" -git clone "$workdir/extended-headers-test-repo/git" "$repodst" >/dev/null 2>&1 +git clone "$workdir/extended-headers-test-repo/git-a" "$repodst" >/dev/null 2>&1 ls "$repodst" diff --git a/tests/git-backup-on-error-multiple-backups.stdout b/tests/git-backup-on-error-multiple-backups.stdout index d918599..9e44fde 100644 --- a/tests/git-backup-on-error-multiple-backups.stdout +++ b/tests/git-backup-on-error-multiple-backups.stdout @@ -1,6 +1,26 @@ -Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME +Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-a and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME DATADIR/work-dir/git-backup-test-repo -DATADIR/work-dir/git-backup-test-repo/git +DATADIR/work-dir/git-backup-test-repo/git-a +DATADIR/work-dir/git-backup-test-repo/git-a/FETCH_HEAD +DATADIR/work-dir/git-backup-test-repo/git-a/HEAD +DATADIR/work-dir/git-backup-test-repo/git-a/branches +DATADIR/work-dir/git-backup-test-repo/git-a/config +DATADIR/work-dir/git-backup-test-repo/git-a/description +DATADIR/work-dir/git-backup-test-repo/git-a/hooks +DATADIR/work-dir/git-backup-test-repo/git-a/info +DATADIR/work-dir/git-backup-test-repo/git-a/info/exclude +DATADIR/work-dir/git-backup-test-repo/git-a/info/refs +DATADIR/work-dir/git-backup-test-repo/git-a/lorry-update-count +DATADIR/work-dir/git-backup-test-repo/git-a/objects +DATADIR/work-dir/git-backup-test-repo/git-a/objects/info +DATADIR/work-dir/git-backup-test-repo/git-a/objects/info/packs +DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack +DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file +DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file +DATADIR/work-dir/git-backup-test-repo/git-a/packed-refs +DATADIR/work-dir/git-backup-test-repo/git-a/refs +DATADIR/work-dir/git-backup-test-repo/git-a/refs/heads +DATADIR/work-dir/git-backup-test-repo/git-a/refs/tags DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/FETCH_HEAD DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/HEAD @@ -21,42 +41,3 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/packed-refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/tags -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/FETCH_HEAD -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/HEAD -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/branches -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/config -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/description -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/exclude -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/refs -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/info -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/info/packs -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/pack -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/packed-refs -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/tags -DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD -DATADIR/work-dir/git-backup-test-repo/git/HEAD -DATADIR/work-dir/git-backup-test-repo/git/branches -DATADIR/work-dir/git-backup-test-repo/git/config -DATADIR/work-dir/git-backup-test-repo/git/description -DATADIR/work-dir/git-backup-test-repo/git/hooks -DATADIR/work-dir/git-backup-test-repo/git/info -DATADIR/work-dir/git-backup-test-repo/git/info/exclude -DATADIR/work-dir/git-backup-test-repo/git/info/refs -DATADIR/work-dir/git-backup-test-repo/git/objects -DATADIR/work-dir/git-backup-test-repo/git/objects/info -DATADIR/work-dir/git-backup-test-repo/git/objects/info/packs -DATADIR/work-dir/git-backup-test-repo/git/objects/pack -DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git/packed-refs -DATADIR/work-dir/git-backup-test-repo/git/refs -DATADIR/work-dir/git-backup-test-repo/git/refs/heads -DATADIR/work-dir/git-backup-test-repo/git/refs/tags diff --git a/tests/git-backup-on-error-single-backup.stdout b/tests/git-backup-on-error-single-backup.stdout index db622ef..d2beacd 100644 --- a/tests/git-backup-on-error-single-backup.stdout +++ b/tests/git-backup-on-error-single-backup.stdout @@ -1,6 +1,26 @@ -Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-pre-update and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail +Mirror of git-backup-test-repo failed, state before mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-a and state after mirror is saved at DATADIR/work-dir/git-backup-test-repo/git-post-fail DATADIR/work-dir/git-backup-test-repo -DATADIR/work-dir/git-backup-test-repo/git +DATADIR/work-dir/git-backup-test-repo/git-a +DATADIR/work-dir/git-backup-test-repo/git-a/FETCH_HEAD +DATADIR/work-dir/git-backup-test-repo/git-a/HEAD +DATADIR/work-dir/git-backup-test-repo/git-a/branches +DATADIR/work-dir/git-backup-test-repo/git-a/config +DATADIR/work-dir/git-backup-test-repo/git-a/description +DATADIR/work-dir/git-backup-test-repo/git-a/hooks +DATADIR/work-dir/git-backup-test-repo/git-a/info +DATADIR/work-dir/git-backup-test-repo/git-a/info/exclude +DATADIR/work-dir/git-backup-test-repo/git-a/info/refs +DATADIR/work-dir/git-backup-test-repo/git-a/lorry-update-count +DATADIR/work-dir/git-backup-test-repo/git-a/objects +DATADIR/work-dir/git-backup-test-repo/git-a/objects/info +DATADIR/work-dir/git-backup-test-repo/git-a/objects/info/packs +DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack +DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file +DATADIR/work-dir/git-backup-test-repo/git-a/objects/pack/pack-file +DATADIR/work-dir/git-backup-test-repo/git-a/packed-refs +DATADIR/work-dir/git-backup-test-repo/git-a/refs +DATADIR/work-dir/git-backup-test-repo/git-a/refs/heads +DATADIR/work-dir/git-backup-test-repo/git-a/refs/tags DATADIR/work-dir/git-backup-test-repo/git-post-fail DATADIR/work-dir/git-backup-test-repo/git-post-fail/FETCH_HEAD DATADIR/work-dir/git-backup-test-repo/git-post-fail/HEAD @@ -21,42 +41,3 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail/packed-refs DATADIR/work-dir/git-backup-test-repo/git-post-fail/refs DATADIR/work-dir/git-backup-test-repo/git-post-fail/refs/heads DATADIR/work-dir/git-backup-test-repo/git-post-fail/refs/tags -DATADIR/work-dir/git-backup-test-repo/git-pre-update -DATADIR/work-dir/git-backup-test-repo/git-pre-update/FETCH_HEAD -DATADIR/work-dir/git-backup-test-repo/git-pre-update/HEAD -DATADIR/work-dir/git-backup-test-repo/git-pre-update/branches -DATADIR/work-dir/git-backup-test-repo/git-pre-update/config -DATADIR/work-dir/git-backup-test-repo/git-pre-update/description -DATADIR/work-dir/git-backup-test-repo/git-pre-update/hooks -DATADIR/work-dir/git-backup-test-repo/git-pre-update/info -DATADIR/work-dir/git-backup-test-repo/git-pre-update/info/exclude -DATADIR/work-dir/git-backup-test-repo/git-pre-update/info/refs -DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects -DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/info -DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/info/packs -DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/pack -DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git-pre-update/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git-pre-update/packed-refs -DATADIR/work-dir/git-backup-test-repo/git-pre-update/refs -DATADIR/work-dir/git-backup-test-repo/git-pre-update/refs/heads -DATADIR/work-dir/git-backup-test-repo/git-pre-update/refs/tags -DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD -DATADIR/work-dir/git-backup-test-repo/git/HEAD -DATADIR/work-dir/git-backup-test-repo/git/branches -DATADIR/work-dir/git-backup-test-repo/git/config -DATADIR/work-dir/git-backup-test-repo/git/description -DATADIR/work-dir/git-backup-test-repo/git/hooks -DATADIR/work-dir/git-backup-test-repo/git/info -DATADIR/work-dir/git-backup-test-repo/git/info/exclude -DATADIR/work-dir/git-backup-test-repo/git/info/refs -DATADIR/work-dir/git-backup-test-repo/git/objects -DATADIR/work-dir/git-backup-test-repo/git/objects/info -DATADIR/work-dir/git-backup-test-repo/git/objects/info/packs -DATADIR/work-dir/git-backup-test-repo/git/objects/pack -DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git/objects/pack/pack-file -DATADIR/work-dir/git-backup-test-repo/git/packed-refs -DATADIR/work-dir/git-backup-test-repo/git/refs -DATADIR/work-dir/git-backup-test-repo/git/refs/heads -DATADIR/work-dir/git-backup-test-repo/git/refs/tags diff --git a/tests/git-single-commit.script b/tests/git-single-commit.script index 7a980b2..9c4813e 100755 --- a/tests/git-single-commit.script +++ b/tests/git-single-commit.script @@ -29,7 +29,7 @@ workdir="$DATADIR/work-dir" # verify that the git repository was set up correctly ( - cd "$workdir/git-test-repo/git/" + cd "$workdir/git-test-repo/git-a/" # list the branches git show-ref | cut -d' ' -f2 diff --git a/tests/hg-incremental.script b/tests/hg-incremental.script index 16ca350..489efa7 100755 --- a/tests/hg-incremental.script +++ b/tests/hg-incremental.script @@ -40,7 +40,7 @@ workdir="$DATADIR/work-dir" # verify that the git repository was created correctly ( - cd "$workdir/hg-test-repo/git/" + cd "$workdir/hg-test-repo/git-b/" # list the branches git show-ref | cut -d' ' -f2 diff --git a/tests/hg-single-commit.script b/tests/hg-single-commit.script index e9814b9..0b95dc9 100755 --- a/tests/hg-single-commit.script +++ b/tests/hg-single-commit.script @@ -29,7 +29,7 @@ workdir="$DATADIR/work-dir" # verify that the git repository was created correctly ( - cd "$workdir/hg-test-repo/git/" + cd "$workdir/hg-test-repo/git-a/" # list the branches git show-ref | cut -d' ' -f2 diff --git a/tests/hg-unnamed-head.script b/tests/hg-unnamed-head.script index 512938f..1811645 100755 --- a/tests/hg-unnamed-head.script +++ b/tests/hg-unnamed-head.script @@ -30,7 +30,7 @@ grep -o 'pruned [0-9]* unreachable commit' < "$logfile" || true # verify that the git repository was created correctly ( - cd "$workdir/hg-test-repo/git/" + cd "$workdir/hg-test-repo/git-a/" # check number of commits generated wc -l hg2git-marks diff --git a/tests/make-tarball.script b/tests/make-tarball.script index 2576447..46618cd 100755 --- a/tests/make-tarball.script +++ b/tests/make-tarball.script @@ -29,4 +29,4 @@ workdir="$DATADIR/work-dir" # verify that we can see the tarball generated of the git tree -test -r "${workdir}/make-tarball-repo-bzip2/git/"*"make_tarball_repo_bzip2.tar" +test -r "${workdir}/make-tarball-repo-bzip2/git-a/"*"make_tarball_repo_bzip2.tar" diff --git a/tests/svn-single-commit.script b/tests/svn-single-commit.script index 32a9cca..533f1dc 100755 --- a/tests/svn-single-commit.script +++ b/tests/svn-single-commit.script @@ -28,7 +28,7 @@ workdir="$DATADIR/work-dir" # verify that the git repository was created successfully ( - cd "$workdir/svn-test-repo/git/" + cd "$workdir/svn-test-repo/git-a/" # list the branches git show-ref | cut -d' ' -f2 diff --git a/tests/tar-single-commit.script b/tests/tar-single-commit.script index 1562446..4d91417 100755 --- a/tests/tar-single-commit.script +++ b/tests/tar-single-commit.script @@ -28,7 +28,7 @@ workdir="$DATADIR/work-dir" # verify that the git repositories were created successfully for FORMAT in "gzip" "bzip2" "lzma"; do - cd "$workdir/tar-test-repo-$FORMAT/git/" + cd "$workdir/tar-test-repo-$FORMAT/git-a/" echo "$FORMAT" -- cgit v1.2.1