lorry: Install and use fudge_user_ids plugin for hg-fast-export

Mercurial allows arbitrary strings as user (committer) ids, while
Git requires a name and email address, and specific punctuation
around the address.

hg-fast-export has some provision for automatically fixing-up invalid
committer and author ids, but it doesn't catch everything.  Its
maintainer does not want to extend this, so we use a plugin instead.

* Add a plugin (fudge_user_ids) that should fix up all invalid ids.
* In setup.py:
  - Compile it at build time
  - Install it under a private data directory (/usr/share/lorry)
  - Clean up the bytecode
* In gitify_hg, check whether hg-fast-export supports plugins, and
  where our plugins are.  If this succeeds, add --plugin-path and
  --plugin options to enable fudge_user_ids.

Closes #11.

3 files changed, 91 insertions, 3 deletions

diff --git a/hg-fast-export/plugins/fudge_user_ids/__init__.py b/hg-fast-export/plugins/fudge_user_ids/__init__.py
new file mode 100644
index 0000000..9e81195
--- /dev/null
+++ b/hg-fast-export/plugins/fudge_user_ids/__init__.py
@@ -0,0 +1,47 @@
+# Fudge committer and author ids that git fast-import considers invalid
+# Copyright 2020 Codethink Ltd
+
+import re
+import sys
+
+from mercurial import templatefilters
+
+
+def build_filter(args):
+    return Filter(args)
+
+
+class Filter:
+    # What git considers valid (see parse_ident() in fast-import.c)
+    _valid_id_re = re.compile(rb'^[^<>]* <[^<>]+>$')
+
+    # Special characters we may need to replace
+    _id_special_re = re.compile(rb'[<>]')
+
+    def __init__(self, args):
+        pass
+
+    def commit_message_filter(self, commit_data):
+        for key in ['author', 'committer']:
+            try:
+                user_id = commit_data[key]
+            except KeyError:
+                continue
+
+            if self._valid_id_re.match(user_id):
+                continue
+
+            name = templatefilters.person(user_id)
+            email = templatefilters.email(user_id)
+
+            # Replace any special characters left in the name and email
+            name = self._id_special_re.sub(b'?', name)
+            email = self._id_special_re.sub(b'?', email)
+
+            commit_data[key] = b'%s <%s>' % (name, email)
+
+            sys.stderr.write(
+                'Replaced %s id "%s" with "%s"\n'
+                % (key,
+                   user_id.decode('utf-8', errors='replace'),
+                   commit_data[key].decode('utf-8', errors='replace')))
diff --git a/lorry b/lorry
index 81fdeb3..0e047db 100755
--- a/lorry
+++ b/lorry
@@ -766,8 +766,30 @@ class Lorry(cliapp.Application):
         self.prune_unreachable_marks(gitdir,
                                      os.path.join(gitdir, 'hg2git-marks'))
 
+        # Enable the fudge_user_ids plugin if possible
+        plugin_options = []
+        exit, out, _ = self.runcmd_unchecked(['hg-fast-export', '--help'])
+        if exit == 0 and b'--plugin' in out:
+            for plugin_path in [
+                    # Check under same directory as us, in case we are
+                    # not yet installed
+                    os.path.join(os.path.dirname(__file__),
+                                 'hg-fast-export/plugins'),
+                    # Try walking from <prefix>/bin/lorry to
+                    # <prefix>/share/lorry/...
+                    os.path.join(os.path.dirname(__file__),
+                                 '../share/lorry/hg-fast-export/plugins')
+                ]:
+                if os.path.exists(plugin_path):
+                    plugin_options += [
+                        '--plugin-path', plugin_path,
+                        '--plugin', 'fudge_user_ids'
+                    ]
+                    break
+
         self.progress('.. fast-exporting into git')
-        self.run_program(['hg-fast-export', '-r', '../hg', '--quiet', '--force'],
+        self.run_program(['hg-fast-export', '-r', '../hg', '--quiet', '--force',
+                          *plugin_options],
                          cwd=gitdir)
 
     def gitify_archive(self, archive_type, project_name, dirname, gitdir, spec):
diff --git a/setup.py b/setup.py
index 52d9e5f..172f11b 100644
--- a/setup.py
+++ b/setup.py
@@ -17,20 +17,26 @@
 '''Setup.py for lorry.'''
 
 
+import compileall
 from distutils.core import setup
 from distutils.cmd import Command
 from distutils.command.build import build
 from distutils.command.clean import clean
+from distutils.command.install import install
+import distutils.dir_util
 import glob
 import os
 import shutil
 import subprocess
 
 
-class GenerateManpage(build):
+class Build(build):
 
     def run(self):
         build.run(self)
+
+        compileall.compile_dir('hg-fast-export/plugins')
+
         print('building manpages')
         for x in ['lorry']:
             with open('%s.1' % x, 'w') as f:
@@ -48,6 +54,7 @@ class Clean(clean):
     ]
     clean_globs = [
         '*/*.py[co]',
+        'hg-fast-export/plugins/*/__pycache__',
     ]
 
     def run(self):
@@ -77,6 +84,17 @@ class Check(Command):
         subprocess.check_call(['./check'])
 
 
+class Install(install):
+    def run(self):
+        install.run(self)
+
+        # Install hg-fast-export plugins in a private directory
+        distutils.dir_util.copy_tree(
+            'hg-fast-export/plugins',
+            os.path.join(self.install_data,
+                         'share/lorry/hg-fast-export/plugins'))
+
+
 setup(name='lorry',
       description='FIXME',
       long_description='''\
@@ -89,7 +107,8 @@ FIXME
                'lorry.zip-importer', 'lorry-ssh-wrapper'],
       data_files=[('share/man/man1', glob.glob('*.[1-8]'))],
       cmdclass={
-          'build': GenerateManpage,
+          'build': Build,
           'check': Check,
           'clean': Clean,
+          'install': Install,
       })