diff options
Diffstat (limited to 'fastimport/processors')
| -rw-r--r-- | fastimport/processors/filter_processor.py | 30 | ||||
| -rw-r--r-- | fastimport/processors/info_processor.py | 58 | ||||
| -rw-r--r-- | fastimport/processors/query_processor.py | 5 |
3 files changed, 56 insertions, 37 deletions
diff --git a/fastimport/processors/filter_processor.py b/fastimport/processors/filter_processor.py index 0ca4472..a252990 100644 --- a/fastimport/processors/filter_processor.py +++ b/fastimport/processors/filter_processor.py @@ -14,7 +14,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """Import processor that filters the input (and doesn't import).""" -from fastimport import ( +from .. import ( commands, helpers, processor, @@ -98,7 +98,7 @@ class FilterProcessor(processor.ImportProcessor): if interesting_filecmds or not self.squash_empty_commits: # If all we have is a single deleteall, skip this commit if len(interesting_filecmds) == 1 and isinstance( - interesting_filecmds[0], commands.FileDeleteAllCommand): + interesting_filecmds[0], commands.FileDeleteAllCommand): pass else: # Remember just the interesting file commands @@ -109,7 +109,7 @@ class FilterProcessor(processor.ImportProcessor): for fc in interesting_filecmds: if isinstance(fc, commands.FileModifyCommand): if (fc.dataref is not None and - not stat.S_ISDIR(fc.mode)): + not stat.S_ISDIR(fc.mode)): self.referenced_blobs.append(fc.dataref) # Update from and merges to refer to commits in the output @@ -149,14 +149,15 @@ class FilterProcessor(processor.ImportProcessor): """Process a FeatureCommand.""" feature = cmd.feature_name if feature not in commands.FEATURE_NAMES: - self.warning("feature %s is not supported - parsing may fail" + self.warning( + "feature %s is not supported - parsing may fail" % (feature,)) # These always pass through self.keep = True def _print_command(self, cmd): """Wrapper to avoid adding unnecessary blank lines.""" - text = helpers.repr_bytes(cmd) + text = bytes(cmd) self.outf.write(text) if not text.endswith(b'\n'): self.outf.write(b'\n') @@ -173,7 +174,7 @@ class FilterProcessor(processor.ImportProcessor): result = [] for fc in filecmd_iter(): if (isinstance(fc, commands.FileModifyCommand) or - isinstance(fc, commands.FileDeleteCommand)): + isinstance(fc, commands.FileDeleteCommand)): if self._path_to_be_kept(fc.path): fc.path = self._adjust_for_new_root(fc.path) else: @@ -185,8 +186,9 @@ class FilterProcessor(processor.ImportProcessor): elif isinstance(fc, commands.FileCopyCommand): fc = self._convert_copy(fc) else: - self.warning("cannot handle FileCommands of class %s - ignoring", - fc.__class__) + self.warning( + "cannot handle FileCommands of class %s - ignoring", + fc.__class__) continue if fc is not None: result.append(fc) @@ -194,11 +196,13 @@ class FilterProcessor(processor.ImportProcessor): def _path_to_be_kept(self, path): """Does the given path pass the filtering criteria?""" - if self.excludes and (path in self.excludes + if self.excludes and ( + path in self.excludes or helpers.is_inside_any(self.excludes, path)): return False if self.includes: - return (path in self.includes + return ( + path in self.includes or helpers.is_inside_any(self.includes, path)) return True @@ -265,7 +269,8 @@ class FilterProcessor(processor.ImportProcessor): # to. Maybe fast-import-info needs to be extended to # remember all renames and a config file can be passed # into here ala fast-import? - self.warning("cannot turn rename of %s into an add of %s yet" % + self.warning( + "cannot turn rename of %s into an add of %s yet" % (old, new)) return None @@ -295,6 +300,7 @@ class FilterProcessor(processor.ImportProcessor): # to. Maybe fast-import-info needs to be extended to # remember all copies and a config file can be passed # into here ala fast-import? - self.warning("cannot turn copy of %s into an add of %s yet" % + self.warning( + "cannot turn copy of %s into an add of %s yet" % (src, dest)) return None diff --git a/fastimport/processors/info_processor.py b/fastimport/processors/info_processor.py index 28c7300..3268e29 100644 --- a/fastimport/processors/info_processor.py +++ b/fastimport/processors/info_processor.py @@ -18,16 +18,14 @@ from __future__ import absolute_import from .. import ( + commands, + processor, reftracker, ) from ..helpers import ( invert_dict, invert_dictset, ) -from fastimport import ( - commands, - processor, - ) import stat @@ -43,8 +41,8 @@ class InfoProcessor(processor.ImportProcessor): """ def __init__(self, params=None, verbose=0, outf=None): - processor.ImportProcessor.__init__(self, params, verbose, - outf=outf) + processor.ImportProcessor.__init__( + self, params, verbose, outf=outf) def pre_process(self): # Init statistics @@ -79,10 +77,13 @@ class InfoProcessor(processor.ImportProcessor): # Dump statistics cmd_names = commands.COMMAND_NAMES fc_names = commands.FILE_COMMAND_NAMES - self._dump_stats_group("Command counts", + self._dump_stats_group( + "Command counts", [(c.decode('utf-8'), self.cmd_counts[c]) for c in cmd_names], str) - self._dump_stats_group("File command counts", - [(c.decode('utf-8'), self.file_cmd_counts[c]) for c in fc_names], str) + self._dump_stats_group( + "File command counts", + [(c.decode('utf-8'), self.file_cmd_counts[c]) for c in fc_names], + str) # Commit stats if self.cmd_counts[b'commit']: @@ -100,7 +101,8 @@ class InfoProcessor(processor.ImportProcessor): 'blobs referenced by SHA': self.sha_blob_references, } self._dump_stats_group("Parent counts", p_items, str) - self._dump_stats_group("Commit analysis", sorted(flags.items()), _found) + self._dump_stats_group( + "Commit analysis", sorted(flags.items()), _found) heads = invert_dictset(self.reftracker.heads) self._dump_stats_group( "Head analysis", @@ -114,10 +116,12 @@ class InfoProcessor(processor.ImportProcessor): # (verbose=2) is specified. The output here for mysql's data can't # be parsed currently so this bit of code needs more work anyhow .. if self.verbose >= 2: - self._dump_stats_group("Rename old paths", + self._dump_stats_group( + "Rename old paths", self.rename_old_paths.items(), len, _iterable_as_config_list) - self._dump_stats_group("Copy source paths", + self._dump_stats_group( + "Copy source paths", self.copy_source_paths.items(), len, _iterable_as_config_list) @@ -126,12 +130,16 @@ class InfoProcessor(processor.ImportProcessor): # In verbose mode, don't list every blob used if self.verbose: del self.blobs['used'] - self._dump_stats_group("Blob usage tracking", - self.blobs.items(), len, _iterable_as_config_list) + self._dump_stats_group( + "Blob usage tracking", + [(k, set([v1.decode() for v1 in v])) + for (k, v) in self.blobs.items()], + len, _iterable_as_config_list) if self.blob_ref_counts: blobs_by_count = invert_dict(self.blob_ref_counts) blob_items = sorted(blobs_by_count.items()) - self._dump_stats_group("Blob reference counts", + self._dump_stats_group( + "Blob reference counts", blob_items, len, _iterable_as_config_list) # Other stats @@ -142,9 +150,9 @@ class InfoProcessor(processor.ImportProcessor): self._dump_stats_group("Reset analysis", reset_stats.items()) def _dump_stats_group(self, title, items, normal_formatter=None, - verbose_formatter=None): + verbose_formatter=None): """Dump a statistics group. - + In verbose mode, do so as a config file so that other processors can load the information if they want to. :param normal_formatter: the callable to apply to the value @@ -210,9 +218,11 @@ class InfoProcessor(processor.ImportProcessor): else: self.sha_blob_references = True elif isinstance(fc, commands.FileRenameCommand): - self.rename_old_paths.setdefault(cmd.id, set()).add(fc.old_path) + self.rename_old_paths.setdefault(cmd.id, set()).add( + fc.old_path) elif isinstance(fc, commands.FileCopyCommand): - self.copy_source_paths.setdefault(cmd.id, set()).add(fc.src_path) + self.copy_source_paths.setdefault(cmd.id, set()).add( + fc.src_path) # Track the heads parents = self.reftracker.track_heads(cmd) @@ -228,7 +238,6 @@ class InfoProcessor(processor.ImportProcessor): # Remember the merges if cmd.merges: - #self.merges.setdefault(cmd.ref, set()).update(cmd.merges) for merge in cmd.merges: if merge in self.merges: self.merges[merge] += 1 @@ -238,7 +247,7 @@ class InfoProcessor(processor.ImportProcessor): def reset_handler(self, cmd): """Process a ResetCommand.""" self.cmd_counts[cmd.name] += 1 - if cmd.ref.startswith('refs/tags/'): + if cmd.ref.startswith(b'refs/tags/'): self.lightweight_tags += 1 else: if cmd.from_ is not None: @@ -254,7 +263,8 @@ class InfoProcessor(processor.ImportProcessor): self.cmd_counts[cmd.name] += 1 feature = cmd.feature_name if feature not in commands.FEATURE_NAMES: - self.warning("feature %s is not supported - parsing may fail" + self.warning( + "feature %s is not supported - parsing may fail" % (feature,)) def _track_blob(self, mark): @@ -270,13 +280,15 @@ class InfoProcessor(processor.ImportProcessor): else: self.blobs['unknown'].add(mark) + def _found(b): """Format a found boolean as a string.""" return ['no', 'found'][b] + def _iterable_as_config_list(s): """Format an iterable as a sequence of comma-separated strings. - + To match what ConfigObj expects, a single item list has a trailing comma. """ items = sorted(s) diff --git a/fastimport/processors/query_processor.py b/fastimport/processors/query_processor.py index a40f2d6..ce93c90 100644 --- a/fastimport/processors/query_processor.py +++ b/fastimport/processors/query_processor.py @@ -17,7 +17,7 @@ from __future__ import print_function -from fastimport import ( +from .. import ( commands, processor, ) @@ -94,5 +94,6 @@ class QueryProcessor(processor.ImportProcessor): """Process a FeatureCommand.""" feature = cmd.feature_name if feature not in commands.FEATURE_NAMES: - self.warning("feature %s is not supported - parsing may fail" + self.warning( + "feature %s is not supported - parsing may fail" % (feature,)) |