Import processors from bzr-fastimport.

5 files changed, 1483 insertions, 0 deletions

diff --git a/fastimport/tests/__init__.py b/fastimport/tests/__init__.py
index 2d80157..3a8e69f 100644
--- a/fastimport/tests/__init__.py
+++ b/fastimport/tests/__init__.py
@@ -26,6 +26,10 @@ def test_suite():
     names = [
         'test_commands',
         'test_errors',
+        'test_filter_processor',
+        'test_helpers',
+        'test_head_tracking',
+        'test_parser',
         ]
     module_names = ['fastimport.tests.' + name for name in names]
     result = unittest.TestSuite()
diff --git a/fastimport/tests/test_filter_processor.py b/fastimport/tests/test_filter_processor.py
new file mode 100644
index 0000000..af107d3
--- /dev/null
+++ b/fastimport/tests/test_filter_processor.py
@@ -0,0 +1,879 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""Test FilterProcessor"""
+
+from cStringIO import StringIO
+
+from testtools import TestCase
+
+from fastimport import (
+    parser,
+    )
+
+from fastimport.processors import (
+    filter_processor,
+    )
+
+
+# A sample input stream containing all (top level) import commands
+_SAMPLE_ALL = \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :2
+committer Joe <joe@example.com> 1234567890 +1000
+data 14
+Initial import
+M 644 :1 COPYING
+checkpoint
+progress first import done
+reset refs/remote/origin/master
+from :2
+tag v0.1
+from :2
+tagger Joe <joe@example.com> 1234567890 +1000
+data 12
+release v0.1
+"""
+
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+_SAMPLE_WITH_DIR = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :2 NEWS
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :101
+M 644 :3 doc/README.txt
+M 644 :4 doc/index.txt
+"""
+
+
+class TestCaseWithFiltering(TestCase):
+
+    def assertFiltering(self, input, params, expected):
+        outf = StringIO()
+        proc = filter_processor.FilterProcessor(
+            params=params)
+        proc.outf = outf
+        s = StringIO(input)
+        p = parser.ImportParser(s)
+        proc.process(p.iter_commands)
+        out = outf.getvalue()
+        self.assertEquals(expected, out)
+
+
+class TestNoFiltering(TestCaseWithFiltering):
+
+    def test_params_not_given(self):
+        self.assertFiltering(_SAMPLE_ALL, None, _SAMPLE_ALL)
+
+    def test_params_are_none(self):
+        params = {'include_paths': None, 'exclude_paths': None}
+        self.assertFiltering(_SAMPLE_ALL, params, _SAMPLE_ALL)
+
+
+class TestIncludePaths(TestCaseWithFiltering):
+
+    def test_file_in_root(self):
+        # Things to note:
+        # * only referenced blobs are retained
+        # * from clause is dropped from the first command
+        params = {'include_paths': ['NEWS']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+M 644 :2 NEWS
+""")
+
+    def test_file_in_subdir(self):
+        #  Additional things to note:
+        # * new root: path is now index.txt, not doc/index.txt
+        # * other files changed in matching commits are excluded
+        params = {'include_paths': ['doc/index.txt']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+M 644 :4 index.txt
+""")
+
+    def test_file_with_changes(self):
+        #  Additional things to note:
+        # * from updated to reference parents in the output
+        params = {'include_paths': ['doc/README.txt']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+""")
+
+    def test_subdir(self):
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+""")
+
+    def test_multiple_files_in_subdir(self):
+        # The new root should be the subdrectory
+        params = {'include_paths': ['doc/README.txt', 'doc/index.txt']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+""")
+
+
+class TestExcludePaths(TestCaseWithFiltering):
+
+    def test_file_in_root(self):
+        params = {'exclude_paths': ['NEWS']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 doc/README.txt
+M 644 :4 doc/index.txt
+""")
+
+    def test_file_in_subdir(self):
+        params = {'exclude_paths': ['doc/README.txt']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+M 644 :2 NEWS
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :101
+M 644 :4 doc/index.txt
+""")
+
+    def test_subdir(self):
+        params = {'exclude_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+M 644 :2 NEWS
+""")
+
+    def test_multple_files(self):
+        params = {'exclude_paths': ['doc/index.txt', 'NEWS']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 doc/README.txt
+""")
+
+
+class TestIncludeAndExcludePaths(TestCaseWithFiltering):
+
+    def test_included_dir_and_excluded_file(self):
+        params = {'include_paths': ['doc/'], 'exclude_paths': ['doc/index.txt']}
+        self.assertFiltering(_SAMPLE_WITH_DIR, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+""")
+
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+#
+# It then renames doc/README.txt => doc/README
+_SAMPLE_WITH_RENAME_INSIDE = _SAMPLE_WITH_DIR + \
+"""commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+R doc/README.txt doc/README
+"""
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+#
+# It then renames doc/README.txt => README
+_SAMPLE_WITH_RENAME_TO_OUTSIDE = _SAMPLE_WITH_DIR + \
+"""commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+R doc/README.txt README
+"""
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+#
+# It then renames NEWS => doc/NEWS
+_SAMPLE_WITH_RENAME_TO_INSIDE = _SAMPLE_WITH_DIR + \
+"""commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+R NEWS doc/NEWS
+"""
+
+class TestIncludePathsWithRenames(TestCaseWithFiltering):
+
+    def test_rename_all_inside(self):
+        # These rename commands ought to be kept but adjusted for the new root
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_RENAME_INSIDE, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+R README.txt README
+""")
+
+    def test_rename_to_outside(self):
+        # These rename commands become deletes
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_RENAME_TO_OUTSIDE, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+D README.txt
+""")
+
+    def test_rename_to_inside(self):
+        # This ought to create a new file but doesn't yet
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_RENAME_TO_INSIDE, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+""")
+
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+#
+# It then copies doc/README.txt => doc/README
+_SAMPLE_WITH_COPY_INSIDE = _SAMPLE_WITH_DIR + \
+"""commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+C doc/README.txt doc/README
+"""
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+#
+# It then copies doc/README.txt => README
+_SAMPLE_WITH_COPY_TO_OUTSIDE = _SAMPLE_WITH_DIR + \
+"""commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+C doc/README.txt README
+"""
+
+# A sample input stream creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+#
+# It then copies NEWS => doc/NEWS
+_SAMPLE_WITH_COPY_TO_INSIDE = _SAMPLE_WITH_DIR + \
+"""commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+C NEWS doc/NEWS
+"""
+
+
+class TestIncludePathsWithCopies(TestCaseWithFiltering):
+
+    def test_copy_all_inside(self):
+        # These copy commands ought to be kept but adjusted for the new root
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_COPY_INSIDE, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 10
+move intro
+from :102
+C README.txt README
+""")
+
+    def test_copy_to_outside(self):
+        # This can be ignored
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_COPY_TO_OUTSIDE, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+""")
+
+    def test_copy_to_inside(self):
+        # This ought to create a new file but doesn't yet
+        params = {'include_paths': ['doc/']}
+        self.assertFiltering(_SAMPLE_WITH_COPY_TO_INSIDE, params, \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 README.txt
+M 644 :4 index.txt
+""")
+
+
+# A sample input stream with deleteall's creating the following tree:
+#
+#  NEWS
+#  doc/README.txt
+#  doc/index.txt
+_SAMPLE_WITH_DELETEALL = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+deleteall
+M 644 :1 doc/README.txt
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+deleteall
+M 644 :3 doc/README.txt
+M 644 :4 doc/index.txt
+"""
+
+
+class TestIncludePathsWithDeleteAll(TestCaseWithFiltering):
+
+    def test_deleteall(self):
+        params = {'include_paths': ['doc/index.txt']}
+        self.assertFiltering(_SAMPLE_WITH_DELETEALL, params, \
+"""blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+deleteall
+M 644 :4 index.txt
+""")
+
+
+_SAMPLE_WITH_TAGS = _SAMPLE_WITH_DIR + \
+"""tag v0.1
+from :100
+tagger d <b@c> 1234798653 +0000
+data 12
+release v0.1
+tag v0.2
+from :102
+tagger d <b@c> 1234798653 +0000
+data 12
+release v0.2
+"""
+
+class TestIncludePathsWithTags(TestCaseWithFiltering):
+
+    def test_tag_retention(self):
+        # If a tag references a commit with a parent we kept,
+        # keep the tag but adjust 'from' accordingly.
+        # Otherwise, delete the tag command.
+        params = {'include_paths': ['NEWS']}
+        self.assertFiltering(_SAMPLE_WITH_TAGS, params, \
+"""blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+M 644 :2 NEWS
+tag v0.2
+from :101
+tagger d <b@c> 1234798653 +0000
+data 12
+release v0.2
+""")
+
+
+_SAMPLE_WITH_RESETS = _SAMPLE_WITH_DIR + \
+"""reset refs/heads/foo
+reset refs/heads/bar
+from :102
+"""
+
+class TestIncludePathsWithResets(TestCaseWithFiltering):
+
+    def test_reset_retention(self):
+        # Resets init'ing a branch (without a from) are passed through.
+        # If a reset references a commit with a parent we kept,
+        # keep the reset but adjust 'from' accordingly.
+        params = {'include_paths': ['NEWS']}
+        self.assertFiltering(_SAMPLE_WITH_RESETS, params, \
+"""blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+M 644 :2 NEWS
+reset refs/heads/foo
+reset refs/heads/bar
+from :101
+""")
diff --git a/fastimport/tests/test_head_tracking.py b/fastimport/tests/test_head_tracking.py
new file mode 100644
index 0000000..7a1ba64
--- /dev/null
+++ b/fastimport/tests/test_head_tracking.py
@@ -0,0 +1,260 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""Test tracking of heads"""
+
+from cStringIO import StringIO
+
+from fastimport import (
+    commands,
+    parser,
+    )
+
+import testtools
+
+from fastimport.reftracker import (
+    RefTracker,
+    )
+
+
+# A sample input stream that only adds files to a branch
+_SAMPLE_MAINLINE = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/master
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :2 NEWS
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :101
+M 644 :3 doc/README.txt
+M 644 :4 doc/index.txt
+"""
+
+# A sample input stream that adds files to two branches
+_SAMPLE_TWO_HEADS = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/mybranch
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :2 NEWS
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 doc/README.txt
+M 644 :4 doc/index.txt
+"""
+
+# A sample input stream that adds files to two branches
+_SAMPLE_TWO_BRANCHES_MERGED = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+blob
+mark :2
+data 17
+Life
+is
+good ...
+commit refs/heads/mybranch
+mark :101
+committer a <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :2 NEWS
+blob
+mark :3
+data 19
+Welcome!
+my friend
+blob
+mark :4
+data 11
+== Docs ==
+commit refs/heads/master
+mark :102
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+M 644 :3 doc/README.txt
+M 644 :4 doc/index.txt
+commit refs/heads/master
+mark :103
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :102
+merge :101
+D doc/index.txt
+"""
+
+# A sample input stream that contains a reset
+_SAMPLE_RESET = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+reset refs/remotes/origin/master
+from :100
+"""
+
+# A sample input stream that contains a reset and more commits
+_SAMPLE_RESET_WITH_MORE_COMMITS = \
+"""blob
+mark :1
+data 9
+Welcome!
+commit refs/heads/master
+mark :100
+committer a <b@c> 1234798653 +0000
+data 4
+test
+M 644 :1 doc/README.txt
+reset refs/remotes/origin/master
+from :100
+commit refs/remotes/origin/master
+mark :101
+committer d <b@c> 1234798653 +0000
+data 8
+test
+ing
+from :100
+D doc/README.txt
+"""
+
+class TestHeadTracking(testtools.TestCase):
+
+    def assertHeads(self, input, expected):
+        s = StringIO(input)
+        p = parser.ImportParser(s)
+        reftracker = RefTracker()
+        for cmd in p.iter_commands():
+            if isinstance(cmd, commands.CommitCommand):
+                reftracker.track_heads(cmd)
+                # eat the file commands
+                list(cmd.file_iter())
+            elif isinstance(cmd, commands.ResetCommand):
+                if cmd.from_ is not None:
+                    reftracker.track_heads_for_ref(cmd.ref, cmd.from_)
+        self.assertEqual(reftracker.heads, expected)
+
+    def test_mainline(self):
+        self.assertHeads(_SAMPLE_MAINLINE, {
+            ':102': set(['refs/heads/master']),
+            })
+
+    def test_two_heads(self):
+        self.assertHeads(_SAMPLE_TWO_HEADS, {
+            ':101': set(['refs/heads/mybranch']),
+            ':102': set(['refs/heads/master']),
+            })
+
+    def test_two_branches_merged(self):
+        self.assertHeads(_SAMPLE_TWO_BRANCHES_MERGED, {
+            ':103': set(['refs/heads/master']),
+            })
+
+    def test_reset(self):
+        self.assertHeads(_SAMPLE_RESET, {
+            ':100': set(['refs/heads/master', 'refs/remotes/origin/master']),
+            })
+
+    def test_reset_with_more_commits(self):
+        self.assertHeads(_SAMPLE_RESET_WITH_MORE_COMMITS, {
+            ':101': set(['refs/remotes/origin/master']),
+            })
diff --git a/fastimport/tests/test_helpers.py b/fastimport/tests/test_helpers.py
new file mode 100644
index 0000000..639e436
--- /dev/null
+++ b/fastimport/tests/test_helpers.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2009 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""Test the helper functions."""
+
+import testtools
+
+from fastimport import (
+    helpers,
+    )
+
+
+class TestCommonDirectory(testtools.TestCase):
+
+    def test_no_paths(self):
+        c = helpers.common_directory(None)
+        self.assertEqual(c, None)
+        c = helpers.common_directory([])
+        self.assertEqual(c, None)
+
+    def test_one_path(self):
+        c = helpers.common_directory(['foo'])
+        self.assertEqual(c, '')
+        c = helpers.common_directory(['foo/'])
+        self.assertEqual(c, 'foo/')
+        c = helpers.common_directory(['foo/bar'])
+        self.assertEqual(c, 'foo/')
+
+    def test_two_paths(self):
+        c = helpers.common_directory(['foo', 'bar'])
+        self.assertEqual(c, '')
+        c = helpers.common_directory(['foo/', 'bar'])
+        self.assertEqual(c, '')
+        c = helpers.common_directory(['foo/', 'foo/bar'])
+        self.assertEqual(c, 'foo/')
+        c = helpers.common_directory(['foo/bar/x', 'foo/bar/y'])
+        self.assertEqual(c, 'foo/bar/')
+        c = helpers.common_directory(['foo/bar/aa_x', 'foo/bar/aa_y'])
+        self.assertEqual(c, 'foo/bar/')
+
+    def test_lots_of_paths(self):
+        c = helpers.common_directory(['foo/bar/x', 'foo/bar/y', 'foo/bar/z'])
+        self.assertEqual(c, 'foo/bar/')
diff --git a/fastimport/tests/test_parser.py b/fastimport/tests/test_parser.py
new file mode 100644
index 0000000..267ec13
--- /dev/null
+++ b/fastimport/tests/test_parser.py
@@ -0,0 +1,284 @@
+# Copyright (C) 2008 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+"""Test the Import parsing"""
+
+import StringIO
+
+import testtools
+
+from fastimport import (
+    errors,
+    parser,
+    )
+
+
+class TestLineBasedParser(testtools.TestCase):
+
+    def test_push_line(self):
+        s = StringIO.StringIO("foo\nbar\nbaz\n")
+        p = parser.LineBasedParser(s)
+        self.assertEqual('foo', p.next_line())
+        self.assertEqual('bar', p.next_line())
+        p.push_line('bar')
+        self.assertEqual('bar', p.next_line())
+        self.assertEqual('baz', p.next_line())
+        self.assertEqual(None, p.next_line())
+
+    def test_read_bytes(self):
+        s = StringIO.StringIO("foo\nbar\nbaz\n")
+        p = parser.LineBasedParser(s)
+        self.assertEqual('fo', p.read_bytes(2))
+        self.assertEqual('o\nb', p.read_bytes(3))
+        self.assertEqual('ar', p.next_line())
+        # Test that the line buffer is ignored
+        p.push_line('bar')
+        self.assertEqual('baz', p.read_bytes(3))
+        # Test missing bytes
+        self.assertRaises(errors.MissingBytes, p.read_bytes, 10)
+
+    def test_read_until(self):
+        # TODO
+        return
+        s = StringIO.StringIO("foo\nbar\nbaz\nabc\ndef\nghi\n")
+        p = parser.LineBasedParser(s)
+        self.assertEqual('foo\nbar', p.read_until('baz'))
+        self.assertEqual('abc', p.next_line())
+        # Test that the line buffer is ignored
+        p.push_line('abc')
+        self.assertEqual('def', p.read_until('ghi'))
+        # Test missing terminator
+        self.assertRaises(errors.MissingTerminator, p.read_until('>>>'))
+
+
+# Sample text
+_sample_import_text = """
+progress completed
+# Test blob formats
+blob
+mark :1
+data 4
+aaaablob
+data 5
+bbbbb
+# Commit formats
+commit refs/heads/master
+mark :2
+committer bugs bunny <bugs@bunny.org> now
+data 14
+initial import
+M 644 inline README
+data 18
+Welcome from bugs
+commit refs/heads/master
+committer <bugs@bunny.org> now
+data 13
+second commit
+from :2
+M 644 inline README
+data 23
+Welcome from bugs, etc.
+# Miscellaneous
+checkpoint
+progress completed
+# Test a commit without sub-commands (bug #351717)
+commit refs/heads/master
+mark :3
+author <bugs@bunny.org> now
+committer <bugs@bunny.org> now
+data 20
+first commit, empty
+# Test a commit with a heredoc-style (delimited_data) messsage (bug #400960)
+commit refs/heads/master
+mark :4
+author <bugs@bunny.org> now
+committer <bugs@bunny.org> now
+data <<EOF
+Commit with heredoc-style message
+EOF
+# Test a "submodule"/tree-reference
+commit refs/heads/master
+mark :5
+author <bugs@bunny.org> now
+committer <bugs@bunny.org> now
+data 15
+submodule test
+M 160000 rev-id tree-id
+# Test features
+feature whatever
+feature foo=bar
+# Test commit with properties
+commit refs/heads/master
+mark :6
+committer <bugs@bunny.org> now
+data 18
+test of properties
+property p1
+property p2 5 hohum
+property p3 16 alpha
+beta
+gamma
+property p4 8 whatever
+# Test a commit with multiple authors
+commit refs/heads/master
+mark :7
+author Fluffy <fluffy@bunny.org> now
+author Daffy <daffy@duck.org> now
+author Donald <donald@duck.org> now
+committer <bugs@bunny.org> now
+data 17
+multi-author test
+"""
+
+
+class TestImportParser(testtools.TestCase):
+
+    def test_iter_commands(self):
+        s = StringIO.StringIO(_sample_import_text)
+        p = parser.ImportParser(s)
+        result = []
+        for cmd in p.iter_commands():
+            result.append(cmd)
+            if cmd.name == 'commit':
+                for fc in cmd.file_iter():
+                    result.append(fc)
+        self.assertEqual(len(result), 17)
+        cmd1 = result.pop(0)
+        self.assertEqual('progress', cmd1.name)
+        self.assertEqual('completed', cmd1.message)
+        cmd2 = result.pop(0)
+        self.assertEqual('blob', cmd2.name)
+        self.assertEqual('1', cmd2.mark)
+        self.assertEqual(':1', cmd2.id)
+        self.assertEqual('aaaa', cmd2.data)
+        self.assertEqual(4, cmd2.lineno)
+        cmd3 = result.pop(0)
+        self.assertEqual('blob', cmd3.name)
+        self.assertEqual('@7', cmd3.id)
+        self.assertEqual(None, cmd3.mark)
+        self.assertEqual('bbbbb', cmd3.data)
+        self.assertEqual(7, cmd3.lineno)
+        cmd4 = result.pop(0)
+        self.assertEqual('commit', cmd4.name)
+        self.assertEqual('2', cmd4.mark)
+        self.assertEqual(':2', cmd4.id)
+        self.assertEqual('initial import', cmd4.message)
+        self.assertEqual('bugs bunny', cmd4.committer[0])
+        self.assertEqual('bugs@bunny.org', cmd4.committer[1])
+        # FIXME: check timestamp and timezone as well
+        self.assertEqual(None, cmd4.author)
+        self.assertEqual(11, cmd4.lineno)
+        self.assertEqual('refs/heads/master', cmd4.ref)
+        self.assertEqual(None, cmd4.from_)
+        self.assertEqual([], cmd4.merges)
+        file_cmd1 = result.pop(0)
+        self.assertEqual('filemodify', file_cmd1.name)
+        self.assertEqual('README', file_cmd1.path)
+        self.assertEqual('file', file_cmd1.kind)
+        self.assertEqual(False, file_cmd1.is_executable)
+        self.assertEqual('Welcome from bugs\n', file_cmd1.data)
+        cmd5 = result.pop(0)
+        self.assertEqual('commit', cmd5.name)
+        self.assertEqual(None, cmd5.mark)
+        self.assertEqual('@19', cmd5.id)
+        self.assertEqual('second commit', cmd5.message)
+        self.assertEqual('', cmd5.committer[0])
+        self.assertEqual('bugs@bunny.org', cmd5.committer[1])
+        # FIXME: check timestamp and timezone as well
+        self.assertEqual(None, cmd5.author)
+        self.assertEqual(19, cmd5.lineno)
+        self.assertEqual('refs/heads/master', cmd5.ref)
+        self.assertEqual(':2', cmd5.from_)
+        self.assertEqual([], cmd5.merges)
+        file_cmd2 = result.pop(0)
+        self.assertEqual('filemodify', file_cmd2.name)
+        self.assertEqual('README', file_cmd2.path)
+        self.assertEqual('file', file_cmd2.kind)
+        self.assertEqual(False, file_cmd2.is_executable)
+        self.assertEqual('Welcome from bugs, etc.', file_cmd2.data)
+        cmd6 = result.pop(0)
+        self.assertEqual(cmd6.name, 'checkpoint')
+        cmd7 = result.pop(0)
+        self.assertEqual('progress', cmd7.name)
+        self.assertEqual('completed', cmd7.message)
+        cmd = result.pop(0)
+        self.assertEqual('commit', cmd.name)
+        self.assertEqual('3', cmd.mark)
+        self.assertEqual(None, cmd.from_)
+        cmd = result.pop(0)
+        self.assertEqual('commit', cmd.name)
+        self.assertEqual('4', cmd.mark)
+        self.assertEqual('Commit with heredoc-style message\n', cmd.message)
+        cmd = result.pop(0)
+        self.assertEqual('commit', cmd.name)
+        self.assertEqual('5', cmd.mark)
+        self.assertEqual('submodule test\n', cmd.message)
+        file_cmd1 = result.pop(0)
+        self.assertEqual('filemodify', file_cmd1.name)
+        self.assertEqual('tree-id', file_cmd1.path)
+        self.assertEqual('tree-reference', file_cmd1.kind)
+        self.assertEqual(False, file_cmd1.is_executable)
+        self.assertEqual("rev-id", file_cmd1.dataref)
+        cmd = result.pop(0)
+        self.assertEqual('feature', cmd.name)
+        self.assertEqual('whatever', cmd.feature_name)
+        self.assertEqual(None, cmd.value)
+        cmd = result.pop(0)
+        self.assertEqual('feature', cmd.name)
+        self.assertEqual('foo', cmd.feature_name)
+        self.assertEqual('bar', cmd.value)
+        cmd = result.pop(0)
+        self.assertEqual('commit', cmd.name)
+        self.assertEqual('6', cmd.mark)
+        self.assertEqual('test of properties', cmd.message)
+        self.assertEqual({
+            'p1': None,
+            'p2': u'hohum',
+            'p3': u'alpha\nbeta\ngamma',
+            'p4': u'whatever',
+            }, cmd.properties)
+        cmd = result.pop(0)
+        self.assertEqual('commit', cmd.name)
+        self.assertEqual('7', cmd.mark)
+        self.assertEqual('multi-author test', cmd.message)
+        self.assertEqual('', cmd.committer[0])
+        self.assertEqual('bugs@bunny.org', cmd.committer[1])
+        self.assertEqual('Fluffy', cmd.author[0])
+        self.assertEqual('fluffy@bunny.org', cmd.author[1])
+        self.assertEqual('Daffy', cmd.more_authors[0][0])
+        self.assertEqual('daffy@duck.org', cmd.more_authors[0][1])
+        self.assertEqual('Donald', cmd.more_authors[1][0])
+        self.assertEqual('donald@duck.org', cmd.more_authors[1][1])
+
+
+class TestStringParsing(testtools.TestCase):
+
+    def test_unquote(self):
+        s = r'hello \"sweet\" wo\\r\tld'
+        self.assertEquals(r'hello "sweet" wo\r' + "\tld",
+            parser._unquote_c_string(s))
+
+
+class TestPathPairParsing(testtools.TestCase):
+
+    def test_path_pair_simple(self):
+        p = parser.ImportParser("")
+        self.assertEqual(['foo', 'bar'], p._path_pair("foo bar"))
+
+    def test_path_pair_spaces_in_first(self):
+        p = parser.ImportParser("")
+        self.assertEqual(['foo bar', 'baz'],
+            p._path_pair('"foo bar" baz'))