Merge branch 'bwh/mercurial-work-around-unnamed-heads' into 'master'

lorry: Prune unreachable commits from hg-fast-export marks file

Closes #7

See merge request CodethinkLabs/lorry/lorry!12

10 files changed, 341 insertions, 1 deletions

diff --git a/lorry b/lorry
index 1989400..b42861b 100755
--- a/lorry
+++ b/lorry
@@ -31,6 +31,8 @@ import email.message
 import email.utils
 import ftplib
 import re
+import subprocess
+import tempfile
 
 import yaml
 
@@ -625,7 +627,13 @@ class Lorry(cliapp.Application):
         if not os.path.exists(gitdir):
             self.needs_aggressive = True
             self.run_program(['git', 'init', '--bare', gitdir])
-        
+
+        # Since there are marks files in existing deployments that
+        # have broken references, fix up the marks file before rather
+        # than after running hg-fast-export
+        self.prune_unreachable_marks(gitdir,
+                                     os.path.join(gitdir, 'hg2git-marks'))
+
         self.progress('.. fast-exporting into git')
         self.run_program(['hg-fast-export', '-r', '../hg', '--quiet', '--force'],
                          cwd=gitdir)
@@ -722,6 +730,53 @@ class Lorry(cliapp.Application):
         if self.settings['verbose']:
             self.output.write('%s\n' % msg)
 
+    def prune_unreachable_marks(self, gitdir, marks_name):
+        if not os.path.exists(marks_name):
+            return
+
+        # Find reachable commits
+        reachable = set()
+        with subprocess.Popen(['git', 'rev-list', '--all'],
+                              cwd=gitdir, stdin=subprocess.DEVNULL,
+                              stdout=subprocess.PIPE,
+                              universal_newlines=True) as rev_list_proc:
+            for line in rev_list_proc.stdout:
+                reachable.add(line.rstrip('\n'))
+
+        # Filter marks file to temporary file
+        mark_re = re.compile(r':(\S+) ([0-9a-f]{40,})\n')
+        marks_temp_fd, marks_temp_name = \
+            tempfile.mkstemp(dir=os.path.dirname(marks_name))
+        n_pruned = 0
+        try:
+            with open(marks_temp_fd, 'w') as marks_out, \
+                 open(marks_name, 'r') as marks_in:
+                for line in marks_in:
+                    match = mark_re.match(line)
+                    if not match:
+                        msg = ('%s: failed to parse line "%s"'
+                               % (marks_name, line.rstrip('\n')))
+                        logging.warning(msg)
+                        self.output.write('%s\n' % msg)
+                        # We don't know whether it should be kept; err
+                        # on the side of caution
+                        marks_out.write(line)
+                    elif match.group(2) in reachable:
+                        marks_out.write(line)
+                    else:
+                        n_pruned += 1
+
+            # On success, replace marks file with temporary file
+            os.rename(marks_temp_name, marks_name)
+
+            if n_pruned:
+                self.progress('%s: pruned %d unreachable commit(s)'
+                              % (marks_name, n_pruned))
+        except:
+            # On failure, delete temporary file
+            os.unlink(marks_temp_name)
+            raise
+
 
 if __name__ == '__main__':
     Lorry(version=__version__).run()
diff --git a/tests/hg-incremental.script b/tests/hg-incremental.script
new file mode 100755
index 0000000..16ca350
--- /dev/null
+++ b/tests/hg-incremental.script
@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# Tests incremental conversion of a simple hg repository to git.
+#
+# Copyright (C) 2012, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+repo="$DATADIR/hg-test-repo"
+
+logfile="$DATADIR/hg-test-repo.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --verbose --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null
+
+# make a second commit
+(
+  cd "$repo"
+  echo "second line" >> test.txt
+  hg commit --user lorry-test-suite --quiet -m "second commit"
+)
+
+"${SRCDIR}/test-lorry" --verbose --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null
+
+# verify that the git repository was created correctly
+(
+  cd "$workdir/hg-test-repo/git/"
+
+  # list the branches
+  git show-ref | cut -d' ' -f2
+
+  # cat the test file
+  git cat-file blob master:test.txt
+
+  # list the commit messages
+  git log --pretty='%s' master
+)
diff --git a/tests/hg-incremental.setup b/tests/hg-incremental.setup
new file mode 100755
index 0000000..1f965b3
--- /dev/null
+++ b/tests/hg-incremental.setup
@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Tests incremental conversion of a simple hg repository to git.
+#
+# Copyright (C) 2012  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+# create the repository
+repo="$DATADIR/hg-test-repo"
+mkdir "$repo"
+cd "$repo"
+hg init --quiet
+
+# add the test file
+echo "first line" > test.txt
+hg add --quiet test.txt
+
+# make a commit
+hg commit --user lorry-test-suite --quiet -m "first commit"
+
+# create the .lorry file for the repository
+cat <<EOF > $DATADIR/hg-test-repo.lorry
+{
+  "hg-test-repo": {
+    "type": "hg",
+    "url": "file://$repo"
+  }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/hg-incremental.stdout b/tests/hg-incremental.stdout
new file mode 100644
index 0000000..dffabe8
--- /dev/null
+++ b/tests/hg-incremental.stdout
@@ -0,0 +1,5 @@
+refs/heads/master
+first line
+second line
+second commit
+first commit
diff --git a/tests/hg-incremental.teardown b/tests/hg-incremental.teardown
new file mode 100755
index 0000000..409c26b
--- /dev/null
+++ b/tests/hg-incremental.teardown
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Tests incremental conversion of a simple hg repository to git.
+#
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+rm -rf "$DATADIR/hg-test-repo" "$DATADIR/hg-test-repo.lorry" "$DATADIR/work-dir"
diff --git a/tests/hg-single-commit.teardown b/tests/hg-single-commit.teardown
new file mode 100755
index 0000000..5a6aa57
--- /dev/null
+++ b/tests/hg-single-commit.teardown
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Tests converting a simple hg repository to git.
+#
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+rm -rf "$DATADIR/hg-test-repo" "$DATADIR/hg-test-repo.lorry" "$DATADIR/work-dir"
diff --git a/tests/hg-unnamed-head.script b/tests/hg-unnamed-head.script
new file mode 100755
index 0000000..512938f
--- /dev/null
+++ b/tests/hg-unnamed-head.script
@@ -0,0 +1,52 @@
+#!/bin/sh
+#
+# Regression test for https://gitlab.com/CodethinkLabs/lorry/lorry/-/issues/7
+#
+# Copyright (C) 2012, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+
+logfile="$DATADIR/hg-test-repo.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --verbose --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null
+grep -o 'pruned [0-9]* unreachable commit' < "$logfile" || true
+
+# verify that the git repository was created correctly
+(
+  cd "$workdir/hg-test-repo/git/"
+
+  # check number of commits generated
+  wc -l hg2git-marks
+
+  # list the branches
+  git show-ref | cut -d' ' -f2
+
+  # cat the test file
+  git cat-file blob master:test.txt
+
+  # list the commit messages
+  git log --pretty='%s' master
+)
+
+# check that we can another incremental conversion succeeds, and
+# that this involves pruning an unreachable commit
+"${SRCDIR}/test-lorry" --verbose --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/hg-test-repo.lorry" > /dev/null 2> /dev/null
+grep -o 'pruned [0-9]* unreachable commit' < "$logfile"
diff --git a/tests/hg-unnamed-head.setup b/tests/hg-unnamed-head.setup
new file mode 100755
index 0000000..31105f5
--- /dev/null
+++ b/tests/hg-unnamed-head.setup
@@ -0,0 +1,58 @@
+#!/bin/sh
+#
+# Regression test for https://gitlab.com/CodethinkLabs/lorry/lorry/-/issues/7
+#
+# Copyright (C) 2012, 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+# create the repository
+repo="$DATADIR/hg-test-repo"
+mkdir "$repo"
+cd "$repo"
+hg init --quiet
+
+# add the test file
+echo "first line" > test.txt
+hg add --quiet test.txt
+
+# make a commit
+hg commit --user lorry-test-suite --quiet -m "first commit"
+
+base=$(hg id -n)
+
+# make a second commit
+echo "second line" >> test.txt
+hg commit --user lorry-test-suite --quiet -m "second commit"
+
+# make a third, divergent, commit
+hg update -r$base
+echo "third line" >> test.txt
+hg commit --user lorry-test-suite --quiet -m "third commit"
+
+# create the .lorry file for the repository
+cat <<EOF > $DATADIR/hg-test-repo.lorry
+{
+  "hg-test-repo": {
+    "type": "hg",
+    "url": "file://$repo"
+  }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/hg-unnamed-head.stdout b/tests/hg-unnamed-head.stdout
new file mode 100644
index 0000000..029b454
--- /dev/null
+++ b/tests/hg-unnamed-head.stdout
@@ -0,0 +1,7 @@
+3 hg2git-marks
+refs/heads/master
+first line
+third line
+third commit
+first commit
+pruned 1 unreachable commit
diff --git a/tests/hg-unnamed-head.teardown b/tests/hg-unnamed-head.teardown
new file mode 100755
index 0000000..8b8cee1
--- /dev/null
+++ b/tests/hg-unnamed-head.teardown
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+# Regression test for https://gitlab.com/CodethinkLabs/lorry/lorry/-/issues/7
+#
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+rm -rf "$DATADIR/hg-test-repo" "$DATADIR/hg-test-repo.lorry" "$DATADIR/work-dir"