From a7f8b1b63a97c66aea6f39342313d7a5b919cfa7 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 21 Sep 2012 17:13:04 +0100 Subject: Fix up Lorry to expect repositories to be bare. This patch makes Lorry always create bare repositories where it can (Note that it cannot for CVS imports) and to create tarballs of bare repositories (if not disabled) which will be more efficient than bundles for creation and cloning. We may be able to disable bundles later. --- tests/bzr-single-commit.script | 2 +- tests/cvs-single-commit.script | 2 +- tests/git-backup-on-error.script | 5 ++-- tests/git-backup-on-error.stdout | 38 ++++---------------------- tests/git-single-commit.script | 2 +- tests/hg-single-commit.script | 2 +- tests/make-tarball.script | 32 ++++++++++++++++++++++ tests/make-tarball.setup | 59 ++++++++++++++++++++++++++++++++++++++++ tests/no-pushspec-pushall.script | 2 +- tests/pushspecs-only.script | 2 +- tests/svn-single-commit.script | 2 +- tests/tar-single-commit.script | 6 ++-- tests/tar-single-commit.setup | 5 ++-- tests/tar-single-commit.stdout | 9 ++++-- 14 files changed, 118 insertions(+), 50 deletions(-) create mode 100755 tests/make-tarball.script create mode 100755 tests/make-tarball.setup (limited to 'tests') diff --git a/tests/bzr-single-commit.script b/tests/bzr-single-commit.script index a870558..51fdb6d 100755 --- a/tests/bzr-single-commit.script +++ b/tests/bzr-single-commit.script @@ -24,7 +24,7 @@ set -e logfile="$DATADIR/bzr-test-repo.log" workdir="$DATADIR/work-dir" -./lorry --pull-only --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/bzr-test-repo.lorry" > /dev/null 2> /dev/null # verify that the git repository was set up correctly diff --git a/tests/cvs-single-commit.script b/tests/cvs-single-commit.script index ad4eca8..e133fae 100755 --- a/tests/cvs-single-commit.script +++ b/tests/cvs-single-commit.script @@ -28,7 +28,7 @@ export USER=root export LOGNAME=$USER export USERNAME=$USER -./lorry --pull-only --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/cvs-test-repo.lorry" > /dev/null 2> /dev/null # verify that the git repository was created successfully diff --git a/tests/git-backup-on-error.script b/tests/git-backup-on-error.script index 134367c..8be1a83 100755 --- a/tests/git-backup-on-error.script +++ b/tests/git-backup-on-error.script @@ -30,16 +30,17 @@ normalize() { DATETIMESPEC='[0-9]*-[0-9]*-[0-9]*-[0-9]*:[0-9]*:[0-9]*' sed -r -e "s|git-pre-update-$DATETIMESPEC|git-pre-update-DATETIME|g" \ -e "s|git-post-fail-$DATETIMESPEC|git-post-fail-DATETIME|g" \ + -e '/hooks\/.*\.sample/d' \ -e "s|$DATADIR|DATADIR|g" "$@" } # mirror some history -./lorry --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \ +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" --bundle=never \ "$DATADIR/git-backup-test-repo.lorry" | normalize # make upstream disappear to cause errors rm -rf "$repo" -if ./lorry --pull-only --log="$logfile" --working-area="$workdir" \ +if ${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/git-backup-test-repo.lorry" --bundle=never 2>/dev/null | \ normalize then diff --git a/tests/git-backup-on-error.stdout b/tests/git-backup-on-error.stdout index 195970e..7e18431 100644 --- a/tests/git-backup-on-error.stdout +++ b/tests/git-backup-on-error.stdout @@ -8,68 +8,40 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/branches DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/config DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/description DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/applypatch-msg.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/commit-msg.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-commit.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-receive.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/post-update.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-applypatch.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-commit.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/pre-rebase.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/prepare-commit-msg.sample -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/hooks/update.sample DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/exclude DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/objects -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/packed-refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads +DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads/master DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/tags DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME +DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/FETCH_HEAD DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/HEAD DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/branches DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/config DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/description DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/applypatch-msg.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/commit-msg.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-commit.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-receive.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/post-update.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-applypatch.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-commit.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/pre-rebase.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/prepare-commit-msg.sample -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/hooks/update.sample DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/exclude DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/refs DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/packed-refs DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads +DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads/master DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/tags +DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD DATADIR/work-dir/git-backup-test-repo/git/HEAD DATADIR/work-dir/git-backup-test-repo/git/branches DATADIR/work-dir/git-backup-test-repo/git/config DATADIR/work-dir/git-backup-test-repo/git/description DATADIR/work-dir/git-backup-test-repo/git/hooks -DATADIR/work-dir/git-backup-test-repo/git/hooks/applypatch-msg.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/commit-msg.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/post-commit.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/post-receive.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/post-update.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-applypatch.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-commit.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/pre-rebase.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/prepare-commit-msg.sample -DATADIR/work-dir/git-backup-test-repo/git/hooks/update.sample DATADIR/work-dir/git-backup-test-repo/git/info DATADIR/work-dir/git-backup-test-repo/git/info/exclude DATADIR/work-dir/git-backup-test-repo/git/info/refs DATADIR/work-dir/git-backup-test-repo/git/objects -DATADIR/work-dir/git-backup-test-repo/git/packed-refs DATADIR/work-dir/git-backup-test-repo/git/refs DATADIR/work-dir/git-backup-test-repo/git/refs/heads +DATADIR/work-dir/git-backup-test-repo/git/refs/heads/master DATADIR/work-dir/git-backup-test-repo/git/refs/tags diff --git a/tests/git-single-commit.script b/tests/git-single-commit.script index 4a6e8b3..b3d8e43 100755 --- a/tests/git-single-commit.script +++ b/tests/git-single-commit.script @@ -24,7 +24,7 @@ set -e logfile="$DATADIR/git-test-repo.log" workdir="$DATADIR/work-dir" -./lorry --pull-only --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/git-test-repo.lorry" > /dev/null 2> /dev/null # verify that the git repository was set up correctly diff --git a/tests/hg-single-commit.script b/tests/hg-single-commit.script index c370190..585fd33 100755 --- a/tests/hg-single-commit.script +++ b/tests/hg-single-commit.script @@ -24,7 +24,7 @@ set -e logfile="$DATADIR/hg-test-repo.log" workdir="$DATADIR/work-dir" -./lorry --verbose --pull-only --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --verbose --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null # verify that the git repository was created correctly diff --git a/tests/make-tarball.script b/tests/make-tarball.script new file mode 100755 index 0000000..8be7b77 --- /dev/null +++ b/tests/make-tarball.script @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Test that we can create a tarball of the git trees. +# +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +set -e + +logfile="$DATADIR/make-tarball.log" +workdir="$DATADIR/work-dir" + +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ + --tarball=first \ + "$DATADIR/make-tarball-repo.lorry" > /dev/null 2> /dev/null + +# verify that we can see the tarball generated of the git tree + +test -r "${workdir}/make-tarball-repo-bzip2/git/"*"make_tarball_repo_bzip2.tar" diff --git a/tests/make-tarball.setup b/tests/make-tarball.setup new file mode 100755 index 0000000..2032610 --- /dev/null +++ b/tests/make-tarball.setup @@ -0,0 +1,59 @@ +#!/bin/sh +# +# Creates gzip/bzip2/lzma tarballs, each with a single file. +# +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +set -e + +# create the original "repository" +repo="$DATADIR/make-tarball-repo" +mkdir "$repo" +echo "first line" > "$repo/test.txt" + +# create the tarballs +cd "$DATADIR" +tar -czf make-tarball-repo.tar.gz "`basename $repo`" +tar -cjf make-tarball-repo.tar.bz2 "`basename $repo`" +tar -cf make-tarball-repo.tar.lzma "`basename $repo`" --lzma + +# create the .lorry file for the tarball "repositories" +cat < $DATADIR/make-tarball-repo.lorry +{ + "make-tarball-repo-gzip": { + "type": "tarball", + "compression": "gzip", + "strip": 1, + "url": "file://$DATADIR/make-tarball-repo.tar.gz" + }, + "make-tarball-repo-bzip2": { + "type": "tarball", + "compression": "bzip2", + "strip": 1, + "url": "file://$DATADIR/make-tarball-repo.tar.bz2" + }, + "make-tarball-repo-lzma": { + "type": "tarball", + "compression": "lzma", + "strip": 1, + "url": "file://$DATADIR/make-tarball-repo.tar.lzma" + } +} +EOF + +# create the working directory +test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir" diff --git a/tests/no-pushspec-pushall.script b/tests/no-pushspec-pushall.script index cc56d8a..7de3a58 100755 --- a/tests/no-pushspec-pushall.script +++ b/tests/no-pushspec-pushall.script @@ -37,7 +37,7 @@ mirror_path="$DATADIR"/git-mirror mkdir -p "$mirror_path" git init --quiet --bare "$mirror_path"/no-pushspec.git -./lorry --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --log="$logfile" --working-area="$workdir" \ --mirror-base-url-push=file://"$mirror_path" \ --mirror-base-url-fetch=file://"$mirror_path" \ "$lorryfile" diff --git a/tests/pushspecs-only.script b/tests/pushspecs-only.script index 787cc38..d7f4c2f 100755 --- a/tests/pushspecs-only.script +++ b/tests/pushspecs-only.script @@ -41,7 +41,7 @@ mirror_path="$DATADIR"/git-mirror mkdir -p "$mirror_path" git init --quiet --bare "$mirror_path"/pushspecs.git -./lorry --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --log="$logfile" --working-area="$workdir" \ --mirror-base-url-push=file://"$mirror_path" \ --mirror-base-url-fetch=file://"$mirror_path" \ "$lorryfile" diff --git a/tests/svn-single-commit.script b/tests/svn-single-commit.script index f1c769c..cff91b0 100755 --- a/tests/svn-single-commit.script +++ b/tests/svn-single-commit.script @@ -23,7 +23,7 @@ set -e logfile="$DATADIR/svn-test-repo.log" workdir="$DATADIR/work-dir" -./lorry --pull-only --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/svn-test-repo.lorry" > /dev/null 2> /dev/null # verify that the git repository was created successfully diff --git a/tests/tar-single-commit.script b/tests/tar-single-commit.script index 623adee..f1d92c5 100755 --- a/tests/tar-single-commit.script +++ b/tests/tar-single-commit.script @@ -20,10 +20,10 @@ set -e -logfile="$DATADIR/svn-test-repo.log" +logfile="$DATADIR/tar-single-commit.log" workdir="$DATADIR/work-dir" -./lorry --pull-only --log="$logfile" --working-area="$workdir" \ +${SRCDIR}/test-lorry --pull-only --log="$logfile" --working-area="$workdir" \ "$DATADIR/tar-test-repo.lorry" > /dev/null 2> /dev/null # verify that the git repositories were created successfully @@ -39,5 +39,5 @@ for FORMAT in "gzip" "bzip2" "lzma"; do git cat-file blob master:test.txt # list the commit messages - git log --pretty='%s' master + git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR," done diff --git a/tests/tar-single-commit.setup b/tests/tar-single-commit.setup index c99450f..bfc99f8 100755 --- a/tests/tar-single-commit.setup +++ b/tests/tar-single-commit.setup @@ -29,7 +29,8 @@ echo "first line" > "$repo/test.txt" cd "$DATADIR" tar -czf tar-test-repo.tar.gz "`basename $repo`" tar -cjf tar-test-repo.tar.bz2 "`basename $repo`" -tar -cf tar-test-repo.tar.lzma "`basename $repo`" --lzma +tar -cf tar-test-repo.tar "`basename $repo`" +xz -z tar-test-repo.tar # create the .lorry file for the tarball "repositories" cat < $DATADIR/tar-test-repo.lorry @@ -50,7 +51,7 @@ cat < $DATADIR/tar-test-repo.lorry "type": "tarball", "compression": "lzma", "strip": 1, - "url": "file://$DATADIR/tar-test-repo.tar.lzma" + "url": "file://$DATADIR/tar-test-repo.tar.xz" } } EOF diff --git a/tests/tar-single-commit.stdout b/tests/tar-single-commit.stdout index 04cec41..32fcdd5 100644 --- a/tests/tar-single-commit.stdout +++ b/tests/tar-single-commit.stdout @@ -1,12 +1,15 @@ gzip refs/heads/master +refs/tags/tar-test-repo first line -Tarball conversion +Imported from DATADIR/work-dir/tar-test-repo-gzip/tar-test-repo.tar.gz. bzip2 refs/heads/master +refs/tags/tar-test-repo first line -Tarball conversion +Imported from DATADIR/work-dir/tar-test-repo-bzip2/tar-test-repo.tar.bz2. lzma refs/heads/master +refs/tags/tar-test-repo first line -Tarball conversion +Imported from DATADIR/work-dir/tar-test-repo-lzma/tar-test-repo.tar.xz. -- cgit v1.2.1 From 291120ecd777de4439b10a07c6f355c7c9af32ed Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Mon, 24 Sep 2012 10:03:13 +0100 Subject: Alter repack so that we do a 'git gc' instead. Git has a garbage collection 'porcelain' command called 'gc' which does more than just repack the repository. Use that in preference to 'git repack' and also configure the repositories so that they don't use too much RAM whilst repacking. Also, we allow gitify_* routines to set self.needs_aggressive on their initial imports so that we aggressively repack the first clone. --- tests/git-backup-on-error.stdout | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tests') diff --git a/tests/git-backup-on-error.stdout b/tests/git-backup-on-error.stdout index 7e18431..b284701 100644 --- a/tests/git-backup-on-error.stdout +++ b/tests/git-backup-on-error.stdout @@ -12,9 +12,9 @@ DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/exclude DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/info/refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/objects +DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/packed-refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads -DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/heads/master DATADIR/work-dir/git-backup-test-repo/git-post-fail-DATETIME/refs/tags DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/FETCH_HEAD @@ -27,9 +27,9 @@ DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/exclude DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/info/refs DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/objects +DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/packed-refs DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads -DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/heads/master DATADIR/work-dir/git-backup-test-repo/git-pre-update-DATETIME/refs/tags DATADIR/work-dir/git-backup-test-repo/git/FETCH_HEAD DATADIR/work-dir/git-backup-test-repo/git/HEAD @@ -41,7 +41,7 @@ DATADIR/work-dir/git-backup-test-repo/git/info DATADIR/work-dir/git-backup-test-repo/git/info/exclude DATADIR/work-dir/git-backup-test-repo/git/info/refs DATADIR/work-dir/git-backup-test-repo/git/objects +DATADIR/work-dir/git-backup-test-repo/git/packed-refs DATADIR/work-dir/git-backup-test-repo/git/refs DATADIR/work-dir/git-backup-test-repo/git/refs/heads -DATADIR/work-dir/git-backup-test-repo/git/refs/heads/master DATADIR/work-dir/git-backup-test-repo/git/refs/tags -- cgit v1.2.1