Add preserve_all_history flag to filter_processor and tests for it

2 files changed, 131 insertions, 5 deletions

diff --git a/fastimport/processors/filter_processor.py b/fastimport/processors/filter_processor.py
index 954c2e8..e007046 100644
--- a/fastimport/processors/filter_processor.py
+++ b/fastimport/processors/filter_processor.py
@@ -36,16 +36,23 @@ class FilterProcessor(processor.ImportProcessor):
 
     * exclude_paths - a list of paths that should not appear in the output
       stream
+
+    * preserve_all_history - if  True filter processeor will be much more conservative
+      w.r.t. history handling -- it will preserve all commits and links between them,
+      including those to unknown revisions. This is primarily usefull for filtering
+      incremental streams
     """
 
     known_params = [
         'include_paths',
         'exclude_paths',
+        'preserve_all_history'
         ]
 
     def pre_process(self):
         self.includes = self.params.get('include_paths')
         self.excludes = self.params.get('exclude_paths')
+        self.preserve_all_history = bool(self.params.get('preserve_all_history'))
         # What's the new root, if any
         self.new_root = helpers.common_directory(self.includes)
         # Buffer of blobs until we know we need them: mark -> cmd
@@ -91,7 +98,7 @@ class FilterProcessor(processor.ImportProcessor):
         """Process a CommitCommand."""
         # These pass through if they meet the filtering conditions
         interesting_filecmds = self._filter_filecommands(cmd.iter_files)
-        if interesting_filecmds:
+        if interesting_filecmds or self.preserve_all_history:
             # If all we have is a single deleteall, skip this commit
             if len(interesting_filecmds) == 1 and isinstance(
                 interesting_filecmds[0], commands.FileDeleteAllCommand):
@@ -109,8 +116,9 @@ class FilterProcessor(processor.ImportProcessor):
                             self.referenced_blobs.append(fc.dataref)
 
                 # Update from and merges to refer to commits in the output
-                cmd.from_ = self._find_interesting_from(cmd.from_)
-                cmd.merges = self._find_interesting_merges(cmd.merges)
+                if not self.preserve_all_history:
+                    cmd.from_ = self._find_interesting_from(cmd.from_)
+                    cmd.merges = self._find_interesting_merges(cmd.merges)
                 self.interesting_commits.add(cmd.id)
 
         # Keep track of the parents
diff --git a/fastimport/tests/test_filter_processor.py b/fastimport/tests/test_filter_processor.py
index ad040d7..4472424 100644
--- a/fastimport/tests/test_filter_processor.py
+++ b/fastimport/tests/test_filter_processor.py
@@ -102,7 +102,6 @@ M 644 :3 doc/README.txt
 M 644 :4 doc/index.txt
 """
 
-
 class TestCaseWithFiltering(TestCase):
 
     def assertFiltering(self, input, params, expected):
@@ -116,7 +115,6 @@ class TestCaseWithFiltering(TestCase):
         out = outf.getvalue()
         self.assertEquals(expected, out)
 
-
 class TestNoFiltering(TestCaseWithFiltering):
 
     def test_params_not_given(self):
@@ -876,3 +874,123 @@ reset refs/heads/foo
 reset refs/heads/bar
 from :101
 """)
+
+
+# A sample input stream containing empty commit
+_SAMPLE_EMPTY_COMMIT = \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :2
+committer Joe <joe@example.com> 1234567890 +1000
+data 14
+Initial import
+M 644 :1 COPYING
+commit refs/heads/master
+mark :3
+committer Joe <joe@example.com> 1234567890 +1000
+data 12
+empty commit
+"""
+
+# A sample input stream containing unresolved from and merge references
+_SAMPLE_FROM_MERGE_COMMIT = \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :3
+committer Joe <joe@example.com> 1234567890 +1000
+data 6
+import
+M 644 :1 COPYING
+blob
+mark :2
+data 4
+bar
+commit refs/heads/master
+mark :4
+committer Joe <joe@example.com> 1234567890 +1000
+data 19
+unknown from commit
+from :999
+M 644 :2 DATA
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <joe@example.com> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+merge :1001
+M 644 :99 DATA2
+"""
+
+class TestPreserveHistoryFlag(TestCaseWithFiltering):
+    
+    def test_squashing_empty_commits(self):
+        params = {'include_paths': None, 'exclude_paths': None}
+        self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :2
+committer Joe <joe@example.com> 1234567890 +1000
+data 14
+Initial import
+M 644 :1 COPYING
+""")
+        
+    def test_keep_empty_commits(self):
+        params = {'include_paths': None, 'exclude_paths': None, 'preserve_all_history': True}
+        self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, _SAMPLE_EMPTY_COMMIT)
+
+    def test_squash_unresolved_references(self):
+        params = {'include_paths': None, 'exclude_paths': None}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \
+"""blob
+mark :1
+data 4
+foo
+commit refs/heads/master
+mark :3
+committer Joe <joe@example.com> 1234567890 +1000
+data 6
+import
+M 644 :1 COPYING
+blob
+mark :2
+data 4
+bar
+commit refs/heads/master
+mark :4
+committer Joe <joe@example.com> 1234567890 +1000
+data 19
+unknown from commit
+M 644 :2 DATA
+blob
+mark :99
+data 4
+bar
+commit refs/heads/master
+mark :5
+committer Joe <joe@example.com> 1234567890 +1000
+data 12
+merge commit
+from :3
+merge :4
+M 644 :99 DATA2
+""")
+        
+    def test_keep_unresolved_from_and_merge(self):
+        params = {'include_paths': None, 'exclude_paths': None, 'preserve_all_history': True}
+        self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, _SAMPLE_FROM_MERGE_COMMIT)