diff options
Diffstat (limited to 'fastimport')
-rw-r--r-- | fastimport/__init__.py | 2 | ||||
-rw-r--r-- | fastimport/commands.py | 61 | ||||
-rw-r--r-- | fastimport/dates.py | 2 | ||||
-rw-r--r-- | fastimport/errors.py | 15 | ||||
-rw-r--r-- | fastimport/helpers.py | 47 | ||||
-rw-r--r-- | fastimport/parser.py | 76 | ||||
-rw-r--r-- | fastimport/processor.py | 4 | ||||
-rw-r--r-- | fastimport/processors/filter_processor.py | 30 | ||||
-rw-r--r-- | fastimport/processors/info_processor.py | 58 | ||||
-rw-r--r-- | fastimport/processors/query_processor.py | 5 | ||||
-rw-r--r-- | fastimport/py.typed | 0 | ||||
-rw-r--r-- | fastimport/reftracker.py | 5 | ||||
-rw-r--r-- | fastimport/tests/test_commands.py | 139 | ||||
-rw-r--r-- | fastimport/tests/test_dates.py | 1 | ||||
-rw-r--r-- | fastimport/tests/test_errors.py | 27 | ||||
-rw-r--r-- | fastimport/tests/test_filter_processor.py | 179 | ||||
-rw-r--r-- | fastimport/tests/test_helpers.py | 3 | ||||
-rw-r--r-- | fastimport/tests/test_info_processor.py | 1 | ||||
-rw-r--r-- | fastimport/tests/test_parser.py | 23 |
19 files changed, 368 insertions, 310 deletions
diff --git a/fastimport/__init__.py b/fastimport/__init__.py index 0b37616..c062341 100644 --- a/fastimport/__init__.py +++ b/fastimport/__init__.py @@ -30,4 +30,4 @@ it can be used by other projects. Use it like so: processor.process(parser.parse()) """ -__version__ = (0, 9, 8) +__version__ = (0, 9, 14) diff --git a/fastimport/commands.py b/fastimport/commands.py index 7f29599..39119c6 100644 --- a/fastimport/commands.py +++ b/fastimport/commands.py @@ -22,12 +22,10 @@ from __future__ import division import re import stat -import sys -from fastimport.helpers import ( +from .helpers import ( newobject as object, utf8_bytes_string, - repr_bytes, ) @@ -43,9 +41,9 @@ GIT_FAST_IMPORT_NEEDS_EXTRA_SPACE_AFTER_QUOTE = False # Lists of command names COMMAND_NAMES = [b'blob', b'checkpoint', b'commit', b'feature', b'progress', - b'reset', b'tag'] + b'reset', b'tag'] FILE_COMMAND_NAMES = [b'filemodify', b'filedelete', b'filecopy', b'filerename', - b'filedeleteall'] + b'filedeleteall'] # Feature names MULTIPLE_AUTHORS_FEATURE = b'multiple-authors' @@ -70,10 +68,7 @@ class ImportCommand(object): return repr(self) def __repr__(self): - if sys.version_info[0] == 2: - return self.__bytes__() - else: - return bytes(self).decode('utf8') + return bytes(self).decode('utf8') def __bytes__(self): raise NotImplementedError( @@ -147,7 +142,8 @@ class CheckpointCommand(ImportCommand): class CommitCommand(ImportCommand): def __init__(self, ref, mark, author, committer, message, from_, - merges, file_iter, lineno=0, more_authors=None, properties=None): + merges, file_iter, lineno=0, more_authors=None, + properties=None): ImportCommand.__init__(self, b'commit') self.ref = ref self.mark = mark @@ -188,7 +184,6 @@ class CommitCommand(ImportCommand): def __bytes__(self): return self.to_string(include_file_contents=True) - def to_string(self, use_features=True, include_file_contents=False): """ @todo the name to_string is ambiguous since the method actually @@ -224,8 +219,8 @@ class CommitCommand(ImportCommand): if self.merges is None: merge_lines = b'' else: - merge_lines = b''.join([b'\nmerge ' + m - for m in self.merges]) + merge_lines = b''.join( + [b'\nmerge ' + m for m in self.merges]) if use_features and self.properties: property_lines = [] for name in sorted(self.properties): @@ -238,11 +233,11 @@ class CommitCommand(ImportCommand): filecommands = b'' else: if include_file_contents: - filecommands = b''.join([b'\n' + repr_bytes(c) - for c in self.iter_files()]) + filecommands = b''.join( + [b'\n' + bytes(c) for c in self.iter_files()]) else: - filecommands = b''.join([b'\n' + str(c) - for c in self.iter_files()]) + filecommands = b''.join( + [b'\n' + str(c) for c in self.iter_files()]) return b''.join([ b'commit ', self.ref, @@ -390,7 +385,9 @@ class FileModifyCommand(FileCommand): elif self.dataref is None: dataref = b'inline' if include_file_contents: - datastr = ('\ndata %d\n' % len(self.data)).encode('ascii') + self.data + datastr = ( + ('\ndata %d\n' % len(self.data)).encode('ascii') + + self.data) else: dataref = self.dataref path = format_path(self.path) @@ -417,9 +414,9 @@ class FileCopyCommand(FileCommand): self.dest_path = check_path(dest_path) def __bytes__(self): - return b' '.join([b'C', - format_path(self.src_path, quote_spaces=True), - format_path(self.dest_path)]) + return b' '.join( + [b'C', format_path(self.src_path, quote_spaces=True), + format_path(self.dest_path)]) class FileRenameCommand(FileCommand): @@ -456,7 +453,7 @@ class NoteModifyCommand(FileCommand): def __bytes__(self): return (b'N inline :' + self.from_ + - ('\ndata %d\n'% len(self.data)).encode('ascii') + self.data) + ('\ndata %d\n' % len(self.data)).encode('ascii') + self.data) def check_path(path): @@ -468,11 +465,8 @@ def check_path(path): if path is None or path == b'' or path.startswith(b'/'): raise ValueError("illegal path '%s'" % path) - if ( - (sys.version_info[0] >= 3 and not isinstance(path, bytes)) and - (sys.version_info[0] == 2 and not isinstance(path, str)) - ): - raise TypeError("illegale type for path '%r'" % path) + if not isinstance(path, bytes): + raise TypeError("illegal type for path '%r'" % path) return path @@ -491,7 +485,7 @@ def format_path(p, quote_spaces=False): def format_who_when(fields): - """Format a tuple of name,email,secs-since-epoch,utc-offset-secs as a string.""" + """Format tuple of name,email,secs-since-epoch,utc-offset-secs as bytes.""" offset = fields[3] if offset < 0: offset_sign = b'-' @@ -500,7 +494,9 @@ def format_who_when(fields): offset_sign = b'+' offset_hours = offset // 3600 offset_minutes = offset // 60 - offset_hours * 60 - offset_str = offset_sign + ('%02d%02d' % (offset_hours, offset_minutes)).encode('ascii') + offset_str = ( + offset_sign + + ('%02d%02d' % (offset_hours, offset_minutes)).encode('ascii')) name = fields[0] if name == b'': @@ -514,7 +510,9 @@ def format_who_when(fields): email = utf8_bytes_string(email) - return b''.join((name, sep, b'<', email, b'> ', ("%d" % fields[2]).encode('ascii'), b' ', offset_str)) + return b''.join( + (name, sep, b'<', email, b'> ', + ("%d" % fields[2]).encode('ascii'), b' ', offset_str)) def format_property(name, value): @@ -525,6 +523,7 @@ def format_property(name, value): result = b'property ' + utf8_name if value is not None: utf8_value = utf8_bytes_string(value) - result += b' ' + ('%d' % len(utf8_value)).encode('ascii') + b' ' + utf8_value + result += (b' ' + ('%d' % len(utf8_value)).encode('ascii') + + b' ' + utf8_value) return result diff --git a/fastimport/dates.py b/fastimport/dates.py index 3deb8e1..e1c660b 100644 --- a/fastimport/dates.py +++ b/fastimport/dates.py @@ -24,7 +24,7 @@ timestamp,timezone where """ import time -from fastimport import errors +from . import errors def parse_raw(s, lineno=0): diff --git a/fastimport/errors.py b/fastimport/errors.py index 7555628..723f294 100644 --- a/fastimport/errors.py +++ b/fastimport/errors.py @@ -18,7 +18,6 @@ # Prefix to messages to show location information _LOCATION_FMT = "line %(lineno)d: " -# ImportError is heavily based on BzrError class ImportError(Exception): """The base exception class for all import processing exceptions.""" @@ -41,7 +40,7 @@ class MissingBytes(ParsingError): """Raised when EOF encountered while expecting to find more bytes.""" _fmt = (_LOCATION_FMT + "Unexpected EOF - expected %(expected)d bytes," - " found %(found)d") + " found %(found)d") def __init__(self, lineno, expected, found): self.expected = expected @@ -53,7 +52,7 @@ class MissingTerminator(ParsingError): """Raised when EOF encountered while expecting to find a terminator.""" _fmt = (_LOCATION_FMT + - "Unexpected EOF - expected '%(terminator)s' terminator") + "Unexpected EOF - expected '%(terminator)s' terminator") def __init__(self, lineno, terminator): self.terminator = terminator @@ -85,7 +84,7 @@ class BadFormat(ParsingError): """Raised when a section is formatted incorrectly.""" _fmt = (_LOCATION_FMT + "Bad format for section %(section)s in " - "command %(cmd)s: found '%(text)s'") + "command %(cmd)s: found '%(text)s'") def __init__(self, lineno, cmd, section, text): self.cmd = cmd @@ -98,7 +97,7 @@ class InvalidTimezone(ParsingError): """Raised when converting a string timezone to a seconds offset.""" _fmt = (_LOCATION_FMT + - "Timezone %(timezone)r could not be converted.%(reason)s") + "Timezone %(timezone)r could not be converted.%(reason)s") def __init__(self, lineno, timezone, reason=None): self.timezone = timezone @@ -153,7 +152,7 @@ class BadRepositorySize(ImportError): """Raised when the repository has an incorrect number of revisions.""" _fmt = ("Bad repository size - %(found)d revisions found, " - "%(expected)d expected") + "%(expected)d expected") def __init__(self, expected, found): self.expected = expected @@ -165,7 +164,7 @@ class BadRestart(ImportError): """Raised when the import stream and id-map do not match up.""" _fmt = ("Bad restart - attempted to skip commit %(commit_id)s " - "but matching revision-id is unknown") + "but matching revision-id is unknown") def __init__(self, commit_id): self.commit_id = commit_id @@ -176,7 +175,7 @@ class UnknownFeature(ImportError): """Raised when an unknown feature is given in the input stream.""" _fmt = ("Unknown feature '%(feature)s' - try a later importer or " - "an earlier data format") + "an earlier data format") def __init__(self, feature): self.feature = feature diff --git a/fastimport/helpers.py b/fastimport/helpers.py index 67072be..b1451f7 100644 --- a/fastimport/helpers.py +++ b/fastimport/helpers.py @@ -14,20 +14,22 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """Miscellaneous useful stuff.""" -import sys def _common_path_and_rest(l1, l2, common=[]): # From http://code.activestate.com/recipes/208993/ - if len(l1) < 1: return (common, l1, l2) - if len(l2) < 1: return (common, l1, l2) - if l1[0] != l2[0]: return (common, l1, l2) + if len(l1) < 1: + return (common, l1, l2) + if len(l2) < 1: + return (common, l1, l2) + if l1[0] != l2[0]: + return (common, l1, l2) return _common_path_and_rest( l1[1:], l2[1:], common + [ - l1[0:1] # return a byte string in python 3 unlike l1[0] that - # would return an integer. + l1[0:1] # return a byte string in python 3 unlike l1[0] that + # would return an integer. ] ) @@ -45,6 +47,7 @@ def common_directory(paths): otherwise the common directory with a trailing / is returned. """ import posixpath + def get_dir_with_slash(path): if path == b'' or path.endswith(b'/'): return path @@ -100,24 +103,10 @@ def is_inside_any(dir_list, fname): def utf8_bytes_string(s): """Convert a string to a bytes string (if necessary, encode in utf8)""" - if sys.version_info[0] == 2: - if isinstance(s, str): - return s - else: - return s.encode('utf8') + if isinstance(s, str): + return bytes(s, encoding='utf8') else: - if isinstance(s, str): - return bytes(s, encoding='utf8') - else: - return s - - -def repr_bytes(obj): - """Return a bytes representation of the object""" - if sys.version_info[0] == 2: - return repr(obj) - else: - return bytes(obj) + return s class newobject(object): @@ -145,7 +134,7 @@ class newobject(object): s = type(self).__str__(self) else: s = str(self) - if isinstance(s, unicode): + if isinstance(s, unicode): # noqa: F821 return s else: return s.decode('utf-8') @@ -177,8 +166,9 @@ class newobject(object): # d = {} # for k, v in iterable: # d[k] = v - # dict(**kwargs) -> new dictionary initialized with the name=value pairs - # in the keyword argument list. For example: dict(one=1, two=2) + # dict(**kwargs) -> new dictionary initialized with the name=value + # pairs in the keyword argument list. + # For example: dict(one=1, two=2) # """ # if len(args) == 0: @@ -216,7 +206,7 @@ def binary_stream(stream): def invert_dictset(d): - """Invert a dictionary with keys matching a set of values, turned into lists.""" + """Invert a dict with keys matching a set of values, turned into lists.""" # Based on recipe from ASPN result = {} for k, c in d.items(): @@ -260,6 +250,3 @@ def get_source_stream(source): else: stream = open(source, "rb") return stream - - - diff --git a/fastimport/parser.py b/fastimport/parser.py index 7fd0a08..ceee1b4 100644 --- a/fastimport/parser.py +++ b/fastimport/parser.py @@ -99,11 +99,11 @@ The grammar is: exact_data ::= 'data' sp declen lf binary_data; - # note: quoted strings are C-style quoting supporting \c for - # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn + # note: quoted strings are C-style quoting supporting \\c for + # common escapes of 'c' (e..g \\n, \\t, \\\\, \\") or \\nnn where nnn # is the signed byte value in octal. Note that the only # characters which must actually be escaped to protect the - # stream formatting is: \, " and LF. Otherwise these values + # stream formatting is: \\, " and LF. Otherwise these values # are UTF8. # ref_str ::= ref; @@ -164,19 +164,17 @@ import re import sys import codecs -from fastimport import ( +from . import ( commands, dates, errors, ) -from fastimport.helpers import ( +from .helpers import ( newobject as object, utf8_bytes_string, ) -## Stream parsing ## - class LineBasedParser(object): def __init__(self, input_stream): @@ -265,7 +263,7 @@ _WHO_RE = re.compile(br'([^<]*)<(.*)>') class ImportParser(LineBasedParser): def __init__(self, input_stream, verbose=False, output=sys.stdout, - user_mapper=None, strict=True): + user_mapper=None, strict=True): """A Parser of import commands. :param input_stream: the file-like object to read from @@ -356,7 +354,7 @@ class ImportParser(LineBasedParser): def _parse_commit(self, ref): """Parse a commit command.""" - lineno = self.lineno + lineno = self.lineno mark = self._get_mark_if_any() author = self._get_user_info(b'commit', b'author', False) more_authors = [] @@ -388,7 +386,8 @@ class ImportParser(LineBasedParser): properties[name] = value else: break - return commands.CommitCommand(ref, mark, author, committer, message, + return commands.CommitCommand( + ref, mark, author, committer, message, from_, merges, list(self.iter_file_commands()), lineno=lineno, more_authors=more_authors, properties=properties) @@ -418,8 +417,8 @@ class ImportParser(LineBasedParser): else: dataref = params[1] data = None - return commands.FileModifyCommand(path, mode, dataref, - data) + return commands.FileModifyCommand( + path, mode, dataref, data) def _parse_reset(self, ref): """Parse a reset command.""" @@ -429,8 +428,8 @@ class ImportParser(LineBasedParser): def _parse_tag(self, name): """Parse a tag command.""" from_ = self._get_from(b'tag') - tagger = self._get_user_info(b'tag', b'tagger', - accept_just_who=True) + tagger = self._get_user_info( + b'tag', b'tagger', accept_just_who=True) message = self._get_data(b'tag', b'message') return commands.TagCommand(name, from_, tagger, message) @@ -479,11 +478,12 @@ class ImportParser(LineBasedParser): return None def _get_user_info(self, cmd, section, required=True, - accept_just_who=False): + accept_just_who=False): """Parse a user section.""" line = self.next_line() if line.startswith(section + b' '): - return self._who_when(line[len(section + b' '):], cmd, section, + return self._who_when( + line[len(section + b' '):], cmd, section, accept_just_who=accept_just_who) elif required: self.abort(errors.MissingSection, cmd, section) @@ -597,7 +597,7 @@ class ImportParser(LineBasedParser): parts[1] = parts[1][1:-1] elif parts[1].startswith(b'"') or parts[1].endswith(b'"'): self.abort(errors.BadFormat, '?', '?', s) - return [_unquote_c_string(s) for s in parts] + return [_unquote_c_string(part) for part in parts] def _mode(self, s): """Check file mode format and parse into an int. @@ -626,8 +626,8 @@ ESCAPE_SEQUENCE_BYTES_RE = re.compile(br''' | \\[0-7]{1,3} # Octal escapes | \\N\{[^}]+\} # Unicode characters by name | \\[\\'"abfnrtv] # Single-character escapes - )''', re.VERBOSE -) + )''', re.VERBOSE) + ESCAPE_SEQUENCE_RE = re.compile(r''' ( \\U........ @@ -636,24 +636,24 @@ ESCAPE_SEQUENCE_RE = re.compile(r''' | \\[0-7]{1,3} | \\N\{[^}]+\} | \\[\\'"abfnrtv] - )''', re.UNICODE | re.VERBOSE -) - -def _unquote_c_string(s): - """replace C-style escape sequences (\n, \", etc.) with real chars.""" - - # doing a s.encode('utf-8').decode('unicode_escape') can return an - # incorrect output with unicode string (both in py2 and py3) the safest way - # is to match the escape sequences and decoding them alone. - def decode_match(match): - return utf8_bytes_string( - codecs.decode(match.group(0), 'unicode-escape') - ) + )''', re.UNICODE | re.VERBOSE) - if sys.version_info[0] >= 3 and isinstance(s, bytes): - return ESCAPE_SEQUENCE_BYTES_RE.sub(decode_match, s) - else: - return ESCAPE_SEQUENCE_RE.sub(decode_match, s) - -Authorship = collections.namedtuple('Authorship', 'name email timestamp timezone') +def _unquote_c_string(s): + """replace C-style escape sequences (\n, \", etc.) with real chars.""" + # doing a s.encode('utf-8').decode('unicode_escape') can return an + # incorrect output with unicode string (both in py2 and py3) the safest way + # is to match the escape sequences and decoding them alone. + def decode_match(match): + return utf8_bytes_string( + codecs.decode(match.group(0), 'unicode-escape') + ) + + if isinstance(s, bytes): + return ESCAPE_SEQUENCE_BYTES_RE.sub(decode_match, s) + else: + return ESCAPE_SEQUENCE_RE.sub(decode_match, s) + + +Authorship = collections.namedtuple( + 'Authorship', 'name email timestamp timezone') diff --git a/fastimport/processor.py b/fastimport/processor.py index 1eb33cb..be4270c 100644 --- a/fastimport/processor.py +++ b/fastimport/processor.py @@ -32,8 +32,8 @@ processors package for examples. import sys import time -from fastimport import errors -from fastimport.helpers import newobject as object +from . import errors +from .helpers import newobject as object class ImportProcessor(object): diff --git a/fastimport/processors/filter_processor.py b/fastimport/processors/filter_processor.py index 0ca4472..a252990 100644 --- a/fastimport/processors/filter_processor.py +++ b/fastimport/processors/filter_processor.py @@ -14,7 +14,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. """Import processor that filters the input (and doesn't import).""" -from fastimport import ( +from .. import ( commands, helpers, processor, @@ -98,7 +98,7 @@ class FilterProcessor(processor.ImportProcessor): if interesting_filecmds or not self.squash_empty_commits: # If all we have is a single deleteall, skip this commit if len(interesting_filecmds) == 1 and isinstance( - interesting_filecmds[0], commands.FileDeleteAllCommand): + interesting_filecmds[0], commands.FileDeleteAllCommand): pass else: # Remember just the interesting file commands @@ -109,7 +109,7 @@ class FilterProcessor(processor.ImportProcessor): for fc in interesting_filecmds: if isinstance(fc, commands.FileModifyCommand): if (fc.dataref is not None and - not stat.S_ISDIR(fc.mode)): + not stat.S_ISDIR(fc.mode)): self.referenced_blobs.append(fc.dataref) # Update from and merges to refer to commits in the output @@ -149,14 +149,15 @@ class FilterProcessor(processor.ImportProcessor): """Process a FeatureCommand.""" feature = cmd.feature_name if feature not in commands.FEATURE_NAMES: - self.warning("feature %s is not supported - parsing may fail" + self.warning( + "feature %s is not supported - parsing may fail" % (feature,)) # These always pass through self.keep = True def _print_command(self, cmd): """Wrapper to avoid adding unnecessary blank lines.""" - text = helpers.repr_bytes(cmd) + text = bytes(cmd) self.outf.write(text) if not text.endswith(b'\n'): self.outf.write(b'\n') @@ -173,7 +174,7 @@ class FilterProcessor(processor.ImportProcessor): result = [] for fc in filecmd_iter(): if (isinstance(fc, commands.FileModifyCommand) or - isinstance(fc, commands.FileDeleteCommand)): + isinstance(fc, commands.FileDeleteCommand)): if self._path_to_be_kept(fc.path): fc.path = self._adjust_for_new_root(fc.path) else: @@ -185,8 +186,9 @@ class FilterProcessor(processor.ImportProcessor): elif isinstance(fc, commands.FileCopyCommand): fc = self._convert_copy(fc) else: - self.warning("cannot handle FileCommands of class %s - ignoring", - fc.__class__) + self.warning( + "cannot handle FileCommands of class %s - ignoring", + fc.__class__) continue if fc is not None: result.append(fc) @@ -194,11 +196,13 @@ class FilterProcessor(processor.ImportProcessor): def _path_to_be_kept(self, path): """Does the given path pass the filtering criteria?""" - if self.excludes and (path in self.excludes + if self.excludes and ( + path in self.excludes or helpers.is_inside_any(self.excludes, path)): return False if self.includes: - return (path in self.includes + return ( + path in self.includes or helpers.is_inside_any(self.includes, path)) return True @@ -265,7 +269,8 @@ class FilterProcessor(processor.ImportProcessor): # to. Maybe fast-import-info needs to be extended to # remember all renames and a config file can be passed # into here ala fast-import? - self.warning("cannot turn rename of %s into an add of %s yet" % + self.warning( + "cannot turn rename of %s into an add of %s yet" % (old, new)) return None @@ -295,6 +300,7 @@ class FilterProcessor(processor.ImportProcessor): # to. Maybe fast-import-info needs to be extended to # remember all copies and a config file can be passed # into here ala fast-import? - self.warning("cannot turn copy of %s into an add of %s yet" % + self.warning( + "cannot turn copy of %s into an add of %s yet" % (src, dest)) return None diff --git a/fastimport/processors/info_processor.py b/fastimport/processors/info_processor.py index 28c7300..3268e29 100644 --- a/fastimport/processors/info_processor.py +++ b/fastimport/processors/info_processor.py @@ -18,16 +18,14 @@ from __future__ import absolute_import from .. import ( + commands, + processor, reftracker, ) from ..helpers import ( invert_dict, invert_dictset, ) -from fastimport import ( - commands, - processor, - ) import stat @@ -43,8 +41,8 @@ class InfoProcessor(processor.ImportProcessor): """ def __init__(self, params=None, verbose=0, outf=None): - processor.ImportProcessor.__init__(self, params, verbose, - outf=outf) + processor.ImportProcessor.__init__( + self, params, verbose, outf=outf) def pre_process(self): # Init statistics @@ -79,10 +77,13 @@ class InfoProcessor(processor.ImportProcessor): # Dump statistics cmd_names = commands.COMMAND_NAMES fc_names = commands.FILE_COMMAND_NAMES - self._dump_stats_group("Command counts", + self._dump_stats_group( + "Command counts", [(c.decode('utf-8'), self.cmd_counts[c]) for c in cmd_names], str) - self._dump_stats_group("File command counts", - [(c.decode('utf-8'), self.file_cmd_counts[c]) for c in fc_names], str) + self._dump_stats_group( + "File command counts", + [(c.decode('utf-8'), self.file_cmd_counts[c]) for c in fc_names], + str) # Commit stats if self.cmd_counts[b'commit']: @@ -100,7 +101,8 @@ class InfoProcessor(processor.ImportProcessor): 'blobs referenced by SHA': self.sha_blob_references, } self._dump_stats_group("Parent counts", p_items, str) - self._dump_stats_group("Commit analysis", sorted(flags.items()), _found) + self._dump_stats_group( + "Commit analysis", sorted(flags.items()), _found) heads = invert_dictset(self.reftracker.heads) self._dump_stats_group( "Head analysis", @@ -114,10 +116,12 @@ class InfoProcessor(processor.ImportProcessor): # (verbose=2) is specified. The output here for mysql's data can't # be parsed currently so this bit of code needs more work anyhow .. if self.verbose >= 2: - self._dump_stats_group("Rename old paths", + self._dump_stats_group( + "Rename old paths", self.rename_old_paths.items(), len, _iterable_as_config_list) - self._dump_stats_group("Copy source paths", + self._dump_stats_group( + "Copy source paths", self.copy_source_paths.items(), len, _iterable_as_config_list) @@ -126,12 +130,16 @@ class InfoProcessor(processor.ImportProcessor): # In verbose mode, don't list every blob used if self.verbose: del self.blobs['used'] - self._dump_stats_group("Blob usage tracking", - self.blobs.items(), len, _iterable_as_config_list) + self._dump_stats_group( + "Blob usage tracking", + [(k, set([v1.decode() for v1 in v])) + for (k, v) in self.blobs.items()], + len, _iterable_as_config_list) if self.blob_ref_counts: blobs_by_count = invert_dict(self.blob_ref_counts) blob_items = sorted(blobs_by_count.items()) - self._dump_stats_group("Blob reference counts", + self._dump_stats_group( + "Blob reference counts", blob_items, len, _iterable_as_config_list) # Other stats @@ -142,9 +150,9 @@ class InfoProcessor(processor.ImportProcessor): self._dump_stats_group("Reset analysis", reset_stats.items()) def _dump_stats_group(self, title, items, normal_formatter=None, - verbose_formatter=None): + verbose_formatter=None): """Dump a statistics group. - + In verbose mode, do so as a config file so that other processors can load the information if they want to. :param normal_formatter: the callable to apply to the value @@ -210,9 +218,11 @@ class InfoProcessor(processor.ImportProcessor): else: self.sha_blob_references = True elif isinstance(fc, commands.FileRenameCommand): - self.rename_old_paths.setdefault(cmd.id, set()).add(fc.old_path) + self.rename_old_paths.setdefault(cmd.id, set()).add( + fc.old_path) elif isinstance(fc, commands.FileCopyCommand): - self.copy_source_paths.setdefault(cmd.id, set()).add(fc.src_path) + self.copy_source_paths.setdefault(cmd.id, set()).add( + fc.src_path) # Track the heads parents = self.reftracker.track_heads(cmd) @@ -228,7 +238,6 @@ class InfoProcessor(processor.ImportProcessor): # Remember the merges if cmd.merges: - #self.merges.setdefault(cmd.ref, set()).update(cmd.merges) for merge in cmd.merges: if merge in self.merges: self.merges[merge] += 1 @@ -238,7 +247,7 @@ class InfoProcessor(processor.ImportProcessor): def reset_handler(self, cmd): """Process a ResetCommand.""" self.cmd_counts[cmd.name] += 1 - if cmd.ref.startswith('refs/tags/'): + if cmd.ref.startswith(b'refs/tags/'): self.lightweight_tags += 1 else: if cmd.from_ is not None: @@ -254,7 +263,8 @@ class InfoProcessor(processor.ImportProcessor): self.cmd_counts[cmd.name] += 1 feature = cmd.feature_name if feature not in commands.FEATURE_NAMES: - self.warning("feature %s is not supported - parsing may fail" + self.warning( + "feature %s is not supported - parsing may fail" % (feature,)) def _track_blob(self, mark): @@ -270,13 +280,15 @@ class InfoProcessor(processor.ImportProcessor): else: self.blobs['unknown'].add(mark) + def _found(b): """Format a found boolean as a string.""" return ['no', 'found'][b] + def _iterable_as_config_list(s): """Format an iterable as a sequence of comma-separated strings. - + To match what ConfigObj expects, a single item list has a trailing comma. """ items = sorted(s) diff --git a/fastimport/processors/query_processor.py b/fastimport/processors/query_processor.py index a40f2d6..ce93c90 100644 --- a/fastimport/processors/query_processor.py +++ b/fastimport/processors/query_processor.py @@ -17,7 +17,7 @@ from __future__ import print_function -from fastimport import ( +from .. import ( commands, processor, ) @@ -94,5 +94,6 @@ class QueryProcessor(processor.ImportProcessor): """Process a FeatureCommand.""" feature = cmd.feature_name if feature not in commands.FEATURE_NAMES: - self.warning("feature %s is not supported - parsing may fail" + self.warning( + "feature %s is not supported - parsing may fail" % (feature,)) diff --git a/fastimport/py.typed b/fastimport/py.typed new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/fastimport/py.typed diff --git a/fastimport/reftracker.py b/fastimport/reftracker.py index 16a5e45..98a7b9f 100644 --- a/fastimport/reftracker.py +++ b/fastimport/reftracker.py @@ -22,7 +22,8 @@ from __future__ import absolute_import class RefTracker(object): def __init__(self): - # Head tracking: last ref, last id per ref & map of commit ids to ref*s* + # Head tracking: last ref, last id per ref & map of commit ids to + # ref*s* self.last_ref = None self.last_ids = {} self.heads = {} @@ -64,5 +65,3 @@ class RefTracker(object): self.heads.setdefault(cmd_id, set()).add(cmd_ref) self.last_ids[cmd_ref] = cmd_id self.last_ref = cmd_ref - - diff --git a/fastimport/tests/test_commands.py b/fastimport/tests/test_commands.py index ccae34a..dabd6a7 100644 --- a/fastimport/tests/test_commands.py +++ b/fastimport/tests/test_commands.py @@ -18,7 +18,6 @@ from unittest import TestCase from fastimport.helpers import ( - repr_bytes, utf8_bytes_string, ) @@ -31,18 +30,18 @@ class TestBlobDisplay(TestCase): def test_blob(self): c = commands.BlobCommand(b"1", b"hello world") - self.assertEqual(b"blob\nmark :1\ndata 11\nhello world", repr_bytes(c)) + self.assertEqual(b"blob\nmark :1\ndata 11\nhello world", bytes(c)) def test_blob_no_mark(self): c = commands.BlobCommand(None, b"hello world") - self.assertEqual(b"blob\ndata 11\nhello world", repr_bytes(c)) + self.assertEqual(b"blob\ndata 11\nhello world", bytes(c)) class TestCheckpointDisplay(TestCase): def test_checkpoint(self): c = commands.CheckpointCommand() - self.assertEqual(b'checkpoint', repr_bytes(c)) + self.assertEqual(b'checkpoint', bytes(c)) class TestCommitDisplay(TestCase): @@ -50,7 +49,8 @@ class TestCommitDisplay(TestCase): def test_commit(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b"refs/heads/master", b"bbb", None, committer, + c = commands.CommitCommand( + b"refs/heads/master", b"bbb", None, committer, b"release v1.0", b":aaa", None, None) self.assertEqual( b"commit refs/heads/master\n" @@ -59,7 +59,7 @@ class TestCommitDisplay(TestCase): b"data 12\n" b"release v1.0\n" b"from :aaa", - repr_bytes(c)) + bytes(c)) def test_commit_unicode_committer(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) @@ -75,28 +75,31 @@ class TestCommitDisplay(TestCase): ) committer = (name, b'test@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b'refs/heads/master', b'bbb', None, committer, + c = commands.CommitCommand( + b'refs/heads/master', b'bbb', None, committer, b'release v1.0', b':aaa', None, None) - self.assertEqual(commit_utf8, repr_bytes(c)) + self.assertEqual(commit_utf8, bytes(c)) def test_commit_no_mark(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b'refs/heads/master', None, None, committer, - b'release v1.0', b':aaa', None, None) + c = commands.CommitCommand( + b'refs/heads/master', None, None, committer, + b'release v1.0', b':aaa', None, None) self.assertEqual( b"commit refs/heads/master\n" b"committer Joe Wong <joe@example.com> 1234567890 -0600\n" b"data 12\n" b"release v1.0\n" b"from :aaa", - repr_bytes(c)) + bytes(c)) def test_commit_no_from(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b"refs/heads/master", b"bbb", None, committer, + c = commands.CommitCommand( + b"refs/heads/master", b"bbb", None, committer, b"release v1.0", None, None, None) self.assertEqual( b"commit refs/heads/master\n" @@ -104,13 +107,14 @@ class TestCommitDisplay(TestCase): b"committer Joe Wong <joe@example.com> 1234567890 -0600\n" b"data 12\n" b"release v1.0", - repr_bytes(c)) + bytes(c)) def test_commit_with_author(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) author = (b'Sue Wong', b'sue@example.com', 1234565432, -6 * 3600) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b'refs/heads/master', b'bbb', author, + c = commands.CommitCommand( + b'refs/heads/master', b'bbb', author, committer, b'release v1.0', b':aaa', None, None) self.assertEqual( b"commit refs/heads/master\n" @@ -120,12 +124,13 @@ class TestCommitDisplay(TestCase): b"data 12\n" b"release v1.0\n" b"from :aaa", - repr_bytes(c)) + bytes(c)) def test_commit_with_merges(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b"refs/heads/master", b"ddd", None, committer, + c = commands.CommitCommand( + b"refs/heads/master", b"ddd", None, committer, b'release v1.0', b":aaa", [b':bbb', b':ccc'], None) self.assertEqual( b"commit refs/heads/master\n" @@ -136,17 +141,18 @@ class TestCommitDisplay(TestCase): b"from :aaa\n" b"merge :bbb\n" b"merge :ccc", - repr_bytes(c)) + bytes(c)) def test_commit_with_filecommands(self): file_cmds = iter([ commands.FileDeleteCommand(b'readme.txt'), - commands.FileModifyCommand(b'NEWS', 0o100644, None, - b'blah blah blah'), + commands.FileModifyCommand( + b'NEWS', 0o100644, None, b'blah blah blah'), ]) # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.CommitCommand(b'refs/heads/master', b'bbb', None, committer, + c = commands.CommitCommand( + b'refs/heads/master', b'bbb', None, committer, b'release v1.0', b':aaa', None, file_cmds) self.assertEqual( b"commit refs/heads/master\n" @@ -159,7 +165,7 @@ class TestCommitDisplay(TestCase): b"M 644 inline NEWS\n" b"data 14\n" b"blah blah blah", - repr_bytes(c)) + bytes(c)) def test_commit_with_more_authors(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) @@ -169,7 +175,8 @@ class TestCommitDisplay(TestCase): (b'Al Smith', b'al@example.com', 1234565432, -6 * 3600), (b'Bill Jones', b'bill@example.com', 1234565432, -6 * 3600), ] - c = commands.CommitCommand(b'refs/heads/master', b'bbb', author, + c = commands.CommitCommand( + b'refs/heads/master', b'bbb', author, committer, b'release v1.0', b':aaa', None, None, more_authors=more_authors) self.assertEqual( @@ -182,7 +189,7 @@ class TestCommitDisplay(TestCase): b"data 12\n" b"release v1.0\n" b"from :aaa", - repr_bytes(c)) + bytes(c)) def test_commit_with_properties(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) @@ -191,7 +198,8 @@ class TestCommitDisplay(TestCase): u'greeting': u'hello', u'planet': u'world', } - c = commands.CommitCommand(b'refs/heads/master', b'bbb', None, + c = commands.CommitCommand( + b'refs/heads/master', b'bbb', None, committer, b'release v1.0', b':aaa', None, None, properties=properties) self.assertEqual( @@ -203,7 +211,7 @@ class TestCommitDisplay(TestCase): b"from :aaa\n" b"property greeting 5 hello\n" b"property planet 5 world", - repr_bytes(c)) + bytes(c)) def test_commit_with_int_mark(self): # user tuple is (name, email, secs-since-epoch, secs-offset-from-utc) @@ -212,7 +220,8 @@ class TestCommitDisplay(TestCase): u'greeting': u'hello', u'planet': u'world', } - c = commands.CommitCommand(b'refs/heads/master', 123, None, + c = commands.CommitCommand( + b'refs/heads/master', 123, None, committer, b'release v1.0', b':aaa', None, None, properties=properties) self.assertEqual( @@ -224,7 +233,8 @@ class TestCommitDisplay(TestCase): b"from :aaa\n" b"property greeting 5 hello\n" b"property planet 5 world", - repr_bytes(c)) + bytes(c)) + class TestCommitCopy(TestCase): @@ -232,7 +242,8 @@ class TestCommitCopy(TestCase): super(TestCommitCopy, self).setUp() file_cmds = iter([ commands.FileDeleteCommand(b'readme.txt'), - commands.FileModifyCommand(b'NEWS', 0o100644, None, b'blah blah blah'), + commands.FileModifyCommand( + b'NEWS', 0o100644, None, b'blah blah blah'), ]) committer = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) @@ -244,45 +255,46 @@ class TestCommitCopy(TestCase): c2 = self.c.copy() self.assertFalse(self.c is c2) - self.assertEqual(repr_bytes(self.c), repr_bytes(c2)) + self.assertEqual(bytes(self.c), bytes(c2)) def test_replace_attr(self): c2 = self.c.copy(mark=b'ccc') self.assertEqual( - repr_bytes(self.c).replace(b'mark :bbb', b'mark :ccc'), - repr_bytes(c2) + bytes(self.c).replace(b'mark :bbb', b'mark :ccc'), + bytes(c2) ) def test_invalid_attribute(self): self.assertRaises(TypeError, self.c.copy, invalid=True) + class TestFeatureDisplay(TestCase): def test_feature(self): c = commands.FeatureCommand(b"dwim") - self.assertEqual(b"feature dwim", repr_bytes(c)) + self.assertEqual(b"feature dwim", bytes(c)) def test_feature_with_value(self): c = commands.FeatureCommand(b"dwim", b"please") - self.assertEqual(b"feature dwim=please", repr_bytes(c)) + self.assertEqual(b"feature dwim=please", bytes(c)) class TestProgressDisplay(TestCase): def test_progress(self): c = commands.ProgressCommand(b"doing foo") - self.assertEqual(b"progress doing foo", repr_bytes(c)) + self.assertEqual(b"progress doing foo", bytes(c)) class TestResetDisplay(TestCase): def test_reset(self): c = commands.ResetCommand(b"refs/tags/v1.0", b":xxx") - self.assertEqual(b"reset refs/tags/v1.0\nfrom :xxx\n", repr_bytes(c)) + self.assertEqual(b"reset refs/tags/v1.0\nfrom :xxx\n", bytes(c)) def test_reset_no_from(self): c = commands.ResetCommand(b'refs/remotes/origin/master', None) - self.assertEqual(b'reset refs/remotes/origin/master', repr_bytes(c)) + self.assertEqual(b'reset refs/remotes/origin/master', bytes(c)) class TestTagDisplay(TestCase): @@ -290,93 +302,100 @@ class TestTagDisplay(TestCase): def test_tag(self): # tagger tuple is (name, email, secs-since-epoch, secs-offset-from-utc) tagger = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.TagCommand(b'refs/tags/v1.0', b':xxx', tagger, b'create v1.0') + c = commands.TagCommand( + b'refs/tags/v1.0', b':xxx', tagger, b'create v1.0') self.assertEqual( b"tag refs/tags/v1.0\n" b"from :xxx\n" b"tagger Joe Wong <joe@example.com> 1234567890 -0600\n" b"data 11\n" b"create v1.0", - repr_bytes(c)) + bytes(c)) def test_tag_no_from(self): tagger = (b'Joe Wong', b'joe@example.com', 1234567890, -6 * 3600) - c = commands.TagCommand(b'refs/tags/v1.0', None, tagger, b'create v1.0') + c = commands.TagCommand( + b'refs/tags/v1.0', None, tagger, b'create v1.0') self.assertEqual( b"tag refs/tags/v1.0\n" b"tagger Joe Wong <joe@example.com> 1234567890 -0600\n" b"data 11\n" b"create v1.0", - repr_bytes(c)) + bytes(c)) class TestFileModifyDisplay(TestCase): def test_filemodify_file(self): c = commands.FileModifyCommand(b'foo/bar', 0o100644, b':23', None) - self.assertEqual(b'M 644 :23 foo/bar', repr_bytes(c)) + self.assertEqual(b'M 644 :23 foo/bar', bytes(c)) def test_filemodify_file_executable(self): c = commands.FileModifyCommand(b'foo/bar', 0o100755, b':23', None) - self.assertEqual(b'M 755 :23 foo/bar', repr_bytes(c)) + self.assertEqual(b'M 755 :23 foo/bar', bytes(c)) def test_filemodify_file_internal(self): - c = commands.FileModifyCommand(b'foo/bar', 0o100644, None, - b'hello world') - self.assertEqual(b'M 644 inline foo/bar\ndata 11\nhello world', repr_bytes(c)) + c = commands.FileModifyCommand( + b'foo/bar', 0o100644, None, b'hello world') + self.assertEqual( + b'M 644 inline foo/bar\ndata 11\nhello world', bytes(c)) def test_filemodify_symlink(self): c = commands.FileModifyCommand(b'foo/bar', 0o120000, None, b'baz') - self.assertEqual(b'M 120000 inline foo/bar\ndata 3\nbaz', repr_bytes(c)) + self.assertEqual( + b'M 120000 inline foo/bar\ndata 3\nbaz', bytes(c)) def test_filemodify_treeref(self): - c = commands.FileModifyCommand(b'tree-info', 0o160000, - b'revision-id-info', None) - self.assertEqual(b'M 160000 revision-id-info tree-info', repr_bytes(c)) + c = commands.FileModifyCommand( + b'tree-info', 0o160000, b'revision-id-info', None) + self.assertEqual( + b'M 160000 revision-id-info tree-info', bytes(c)) class TestFileDeleteDisplay(TestCase): def test_filedelete(self): c = commands.FileDeleteCommand(b'foo/bar') - self.assertEqual(b'D foo/bar', repr_bytes(c)) + self.assertEqual(b'D foo/bar', bytes(c)) class TestFileCopyDisplay(TestCase): def test_filecopy(self): c = commands.FileCopyCommand(b'foo/bar', b'foo/baz') - self.assertEqual(b'C foo/bar foo/baz', repr_bytes(c)) + self.assertEqual(b'C foo/bar foo/baz', bytes(c)) def test_filecopy_quoted(self): # Check the first path is quoted if it contains spaces c = commands.FileCopyCommand(b'foo/b a r', b'foo/b a z') - self.assertEqual(b'C "foo/b a r" foo/b a z', repr_bytes(c)) + self.assertEqual(b'C "foo/b a r" foo/b a z', bytes(c)) class TestFileRenameDisplay(TestCase): def test_filerename(self): c = commands.FileRenameCommand(b'foo/bar', b'foo/baz') - self.assertEqual(b'R foo/bar foo/baz', repr_bytes(c)) + self.assertEqual(b'R foo/bar foo/baz', bytes(c)) def test_filerename_quoted(self): # Check the first path is quoted if it contains spaces c = commands.FileRenameCommand(b'foo/b a r', b'foo/b a z') - self.assertEqual(b'R "foo/b a r" foo/b a z', repr_bytes(c)) + self.assertEqual(b'R "foo/b a r" foo/b a z', bytes(c)) class TestFileDeleteAllDisplay(TestCase): def test_filedeleteall(self): c = commands.FileDeleteAllCommand() - self.assertEqual(b'deleteall', repr_bytes(c)) + self.assertEqual(b'deleteall', bytes(c)) + class TestNotesDisplay(TestCase): def test_noteonly(self): c = commands.NoteModifyCommand(b'foo', b'A basic note') - self.assertEqual(b'N inline :foo\ndata 12\nA basic note', repr_bytes(c)) + self.assertEqual( + b'N inline :foo\ndata 12\nA basic note', bytes(c)) def test_notecommit(self): committer = (b'Ed Mund', b'ed@example.org', 1234565432, 0) @@ -443,15 +462,17 @@ Notes added by 'git notes add' N inline :1 data 10 Test test -""", b''.join([repr_bytes(s) for s in commits])) +""", b''.join([bytes(s) for s in commits])) class TestPathChecking(TestCase): def test_filemodify_path_checking(self): - self.assertRaises(ValueError, commands.FileModifyCommand, b'', + self.assertRaises( + ValueError, commands.FileModifyCommand, b'', 0o100644, None, b'text') - self.assertRaises(ValueError, commands.FileModifyCommand, None, + self.assertRaises( + ValueError, commands.FileModifyCommand, None, 0o100644, None, b'text') def test_filedelete_path_checking(self): diff --git a/fastimport/tests/test_dates.py b/fastimport/tests/test_dates.py index f1ccd67..cc2617f 100644 --- a/fastimport/tests/test_dates.py +++ b/fastimport/tests/test_dates.py @@ -21,6 +21,7 @@ from fastimport import ( dates, ) + class ParseTzTests(TestCase): def test_parse_tz_utc(self): diff --git a/fastimport/tests/test_errors.py b/fastimport/tests/test_errors.py index 4fc7dcd..895f783 100644 --- a/fastimport/tests/test_errors.py +++ b/fastimport/tests/test_errors.py @@ -25,27 +25,28 @@ class TestErrors(TestCase): def test_MissingBytes(self): e = errors.MissingBytes(99, 10, 8) - self.assertEqual("line 99: Unexpected EOF - expected 10 bytes, found 8", + self.assertEqual( + "line 99: Unexpected EOF - expected 10 bytes, found 8", str(e)) def test_MissingTerminator(self): e = errors.MissingTerminator(99, '---') - self.assertEqual("line 99: Unexpected EOF - expected '---' terminator", + self.assertEqual( + "line 99: Unexpected EOF - expected '---' terminator", str(e)) def test_InvalidCommand(self): e = errors.InvalidCommand(99, 'foo') - self.assertEqual("line 99: Invalid command 'foo'", - str(e)) + self.assertEqual("line 99: Invalid command 'foo'", str(e)) def test_MissingSection(self): e = errors.MissingSection(99, 'foo', 'bar') - self.assertEqual("line 99: Command foo is missing section bar", - str(e)) + self.assertEqual("line 99: Command foo is missing section bar", str(e)) def test_BadFormat(self): e = errors.BadFormat(99, 'foo', 'bar', 'xyz') - self.assertEqual("line 99: Bad format for section bar in " + self.assertEqual( + "line 99: Bad format for section bar in " "command foo: found 'xyz'", str(e)) @@ -53,14 +54,15 @@ class TestErrors(TestCase): e = errors.InvalidTimezone(99, 'aa:bb') self.assertEqual('aa:bb', e.timezone) self.assertEqual('', e.reason) - self.assertEqual("line 99: Timezone 'aa:bb' could not be converted.", + self.assertEqual( + "line 99: Timezone 'aa:bb' could not be converted.", str(e)) e = errors.InvalidTimezone(99, 'aa:bb', 'Non-numeric hours') self.assertEqual('aa:bb', e.timezone) self.assertEqual(' Non-numeric hours', e.reason) - self.assertEqual("line 99: Timezone 'aa:bb' could not be converted." - " Non-numeric hours", - str(e)) + self.assertEqual( + "line 99: Timezone 'aa:bb' could not be converted." + " Non-numeric hours", str(e)) def test_UnknownDateFormat(self): e = errors.UnknownDateFormat('aaa') @@ -72,5 +74,6 @@ class TestErrors(TestCase): def test_UnknownFeature(self): e = errors.UnknownFeature('aaa') - self.assertEqual("Unknown feature 'aaa' - try a later importer or " + self.assertEqual( + "Unknown feature 'aaa' - try a later importer or " "an earlier data format", str(e)) diff --git a/fastimport/tests/test_filter_processor.py b/fastimport/tests/test_filter_processor.py index 809bdc8..7659db1 100644 --- a/fastimport/tests/test_filter_processor.py +++ b/fastimport/tests/test_filter_processor.py @@ -28,8 +28,8 @@ from fastimport.processors import ( # A sample input stream containing all (top level) import commands -_SAMPLE_ALL = \ -b"""blob +_SAMPLE_ALL = b"""\ +blob mark :1 data 4 foo @@ -56,8 +56,8 @@ release v0.1 # NEWS # doc/README.txt # doc/index.txt -_SAMPLE_WITH_DIR = \ -b"""blob +_SAMPLE_WITH_DIR = b"""\ +blob mark :1 data 9 Welcome! @@ -101,6 +101,7 @@ M 644 :3 doc/README.txt M 644 :4 doc/index.txt """ + class TestCaseWithFiltering(TestCase): def assertFiltering(self, input_stream, params, expected): @@ -114,6 +115,7 @@ class TestCaseWithFiltering(TestCase): out = outf.getvalue() self.assertEqual(expected, out) + class TestNoFiltering(TestCaseWithFiltering): def test_params_not_given(self): @@ -131,8 +133,8 @@ class TestIncludePaths(TestCaseWithFiltering): # * only referenced blobs are retained # * from clause is dropped from the first command params = {b'include_paths': [b'NEWS']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :2 data 17 Life @@ -152,8 +154,8 @@ M 644 :2 NEWS # * new root: path is now index.txt, not doc/index.txt # * other files changed in matching commits are excluded params = {b'include_paths': [b'doc/index.txt']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :4 data 11 == Docs == @@ -170,8 +172,8 @@ M 644 :4 index.txt # Additional things to note: # * from updated to reference parents in the output params = {b'include_paths': [b'doc/README.txt']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :1 data 9 Welcome! @@ -198,8 +200,8 @@ M 644 :3 README.txt def test_subdir(self): params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :1 data 9 Welcome! @@ -232,8 +234,8 @@ M 644 :4 index.txt def test_multiple_files_in_subdir(self): # The new root should be the subdrectory params = {b'include_paths': [b'doc/README.txt', b'doc/index.txt']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :1 data 9 Welcome! @@ -268,8 +270,8 @@ class TestExcludePaths(TestCaseWithFiltering): def test_file_in_root(self): params = {b'exclude_paths': [b'NEWS']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :1 data 9 Welcome! @@ -301,8 +303,8 @@ M 644 :4 doc/index.txt def test_file_in_subdir(self): params = {b'exclude_paths': [b'doc/README.txt']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :2 data 17 Life @@ -331,8 +333,8 @@ M 644 :4 doc/index.txt def test_subdir(self): params = {b'exclude_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :2 data 17 Life @@ -349,8 +351,8 @@ M 644 :2 NEWS def test_multple_files(self): params = {b'exclude_paths': [b'doc/index.txt', b'NEWS']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :1 data 9 Welcome! @@ -379,9 +381,11 @@ M 644 :3 doc/README.txt class TestIncludeAndExcludePaths(TestCaseWithFiltering): def test_included_dir_and_excluded_file(self): - params = {b'include_paths': [b'doc/'], b'exclude_paths': [b'doc/index.txt']} - self.assertFiltering(_SAMPLE_WITH_DIR, params, \ -b"""blob + params = { + b'include_paths': [b'doc/'], + b'exclude_paths': [b'doc/index.txt']} + self.assertFiltering(_SAMPLE_WITH_DIR, params, b"""\ +blob mark :1 data 9 Welcome! @@ -414,8 +418,8 @@ M 644 :3 README.txt # doc/index.txt # # It then renames doc/README.txt => doc/README -_SAMPLE_WITH_RENAME_INSIDE = _SAMPLE_WITH_DIR + \ -b"""commit refs/heads/master +_SAMPLE_WITH_RENAME_INSIDE = _SAMPLE_WITH_DIR + b"""\ +commit refs/heads/master mark :103 committer d <b@c> 1234798653 +0000 data 10 @@ -431,8 +435,8 @@ R doc/README.txt doc/README # doc/index.txt # # It then renames doc/README.txt => README -_SAMPLE_WITH_RENAME_TO_OUTSIDE = _SAMPLE_WITH_DIR + \ -b"""commit refs/heads/master +_SAMPLE_WITH_RENAME_TO_OUTSIDE = _SAMPLE_WITH_DIR + b"""\ +commit refs/heads/master mark :103 committer d <b@c> 1234798653 +0000 data 10 @@ -448,8 +452,8 @@ R doc/README.txt README # doc/index.txt # # It then renames NEWS => doc/NEWS -_SAMPLE_WITH_RENAME_TO_INSIDE = _SAMPLE_WITH_DIR + \ -b"""commit refs/heads/master +_SAMPLE_WITH_RENAME_TO_INSIDE = _SAMPLE_WITH_DIR + b"""\ +commit refs/heads/master mark :103 committer d <b@c> 1234798653 +0000 data 10 @@ -458,13 +462,14 @@ from :102 R NEWS doc/NEWS """ + class TestIncludePathsWithRenames(TestCaseWithFiltering): def test_rename_all_inside(self): # These rename commands ought to be kept but adjusted for the new root params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_RENAME_INSIDE, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_RENAME_INSIDE, params, b"""\ +blob mark :1 data 9 Welcome! @@ -504,8 +509,8 @@ R README.txt README def test_rename_to_outside(self): # These rename commands become deletes params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_RENAME_TO_OUTSIDE, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_RENAME_TO_OUTSIDE, params, b"""\ +blob mark :1 data 9 Welcome! @@ -545,8 +550,8 @@ D README.txt def test_rename_to_inside(self): # This ought to create a new file but doesn't yet params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_RENAME_TO_INSIDE, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_RENAME_TO_INSIDE, params, b"""\ +blob mark :1 data 9 Welcome! @@ -584,8 +589,8 @@ M 644 :4 index.txt # doc/index.txt # # It then copies doc/README.txt => doc/README -_SAMPLE_WITH_COPY_INSIDE = _SAMPLE_WITH_DIR + \ -b"""commit refs/heads/master +_SAMPLE_WITH_COPY_INSIDE = _SAMPLE_WITH_DIR + b"""\ +commit refs/heads/master mark :103 committer d <b@c> 1234798653 +0000 data 10 @@ -601,8 +606,8 @@ C doc/README.txt doc/README # doc/index.txt # # It then copies doc/README.txt => README -_SAMPLE_WITH_COPY_TO_OUTSIDE = _SAMPLE_WITH_DIR + \ -b"""commit refs/heads/master +_SAMPLE_WITH_COPY_TO_OUTSIDE = _SAMPLE_WITH_DIR + b"""\ +commit refs/heads/master mark :103 committer d <b@c> 1234798653 +0000 data 10 @@ -618,8 +623,8 @@ C doc/README.txt README # doc/index.txt # # It then copies NEWS => doc/NEWS -_SAMPLE_WITH_COPY_TO_INSIDE = _SAMPLE_WITH_DIR + \ -b"""commit refs/heads/master +_SAMPLE_WITH_COPY_TO_INSIDE = _SAMPLE_WITH_DIR + b"""\ +commit refs/heads/master mark :103 committer d <b@c> 1234798653 +0000 data 10 @@ -634,8 +639,8 @@ class TestIncludePathsWithCopies(TestCaseWithFiltering): def test_copy_all_inside(self): # These copy commands ought to be kept but adjusted for the new root params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_COPY_INSIDE, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_COPY_INSIDE, params, b"""\ +blob mark :1 data 9 Welcome! @@ -675,8 +680,8 @@ C README.txt README def test_copy_to_outside(self): # This can be ignored params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_COPY_TO_OUTSIDE, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_COPY_TO_OUTSIDE, params, b"""\ +blob mark :1 data 9 Welcome! @@ -709,8 +714,8 @@ M 644 :4 index.txt def test_copy_to_inside(self): # This ought to create a new file but doesn't yet params = {b'include_paths': [b'doc/']} - self.assertFiltering(_SAMPLE_WITH_COPY_TO_INSIDE, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_COPY_TO_INSIDE, params, b"""\ +blob mark :1 data 9 Welcome! @@ -746,8 +751,8 @@ M 644 :4 index.txt # NEWS # doc/README.txt # doc/index.txt -_SAMPLE_WITH_DELETEALL = \ -b"""blob +_SAMPLE_WITH_DELETEALL = b"""\ +blob mark :1 data 9 Welcome! @@ -784,8 +789,8 @@ class TestIncludePathsWithDeleteAll(TestCaseWithFiltering): def test_deleteall(self): params = {b'include_paths': [b'doc/index.txt']} - self.assertFiltering(_SAMPLE_WITH_DELETEALL, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_DELETEALL, params, b"""\ +blob mark :4 data 11 == Docs == @@ -801,8 +806,8 @@ M 644 :4 index.txt """) -_SAMPLE_WITH_TAGS = _SAMPLE_WITH_DIR + \ -b"""tag v0.1 +_SAMPLE_WITH_TAGS = _SAMPLE_WITH_DIR + b"""\ +tag v0.1 from :100 tagger d <b@c> 1234798653 +0000 data 12 @@ -814,6 +819,7 @@ data 12 release v0.2 """ + class TestIncludePathsWithTags(TestCaseWithFiltering): def test_tag_retention(self): @@ -821,8 +827,8 @@ class TestIncludePathsWithTags(TestCaseWithFiltering): # keep the tag but adjust 'from' accordingly. # Otherwise, delete the tag command. params = {b'include_paths': [b'NEWS']} - self.assertFiltering(_SAMPLE_WITH_TAGS, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_TAGS, params, b"""\ +blob mark :2 data 17 Life @@ -843,12 +849,13 @@ release v0.2 """) -_SAMPLE_WITH_RESETS = _SAMPLE_WITH_DIR + \ -b"""reset refs/heads/foo +_SAMPLE_WITH_RESETS = _SAMPLE_WITH_DIR + b"""\ +reset refs/heads/foo reset refs/heads/bar from :102 """ + class TestIncludePathsWithResets(TestCaseWithFiltering): def test_reset_retention(self): @@ -856,8 +863,8 @@ class TestIncludePathsWithResets(TestCaseWithFiltering): # If a reset references a commit with a parent we kept, # keep the reset but adjust 'from' accordingly. params = {b'include_paths': [b'NEWS']} - self.assertFiltering(_SAMPLE_WITH_RESETS, params, \ -b"""blob + self.assertFiltering(_SAMPLE_WITH_RESETS, params, b"""\ +blob mark :2 data 17 Life @@ -877,8 +884,8 @@ from :101 # A sample input stream containing empty commit -_SAMPLE_EMPTY_COMMIT = \ -b"""blob +_SAMPLE_EMPTY_COMMIT = b"""\ +blob mark :1 data 4 foo @@ -896,8 +903,8 @@ empty commit """ # A sample input stream containing unresolved from and merge references -_SAMPLE_FROM_MERGE_COMMIT = \ -b"""blob +_SAMPLE_FROM_MERGE_COMMIT = b"""\ +blob mark :1 data 4 foo @@ -933,12 +940,13 @@ merge :1001 M 644 :99 data/DATA2 """ + class TestSquashEmptyCommitsFlag(TestCaseWithFiltering): def test_squash_empty_commit(self): params = {b'include_paths': None, b'exclude_paths': None} - self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, \ -b"""blob + self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, b"""\ +blob mark :1 data 4 foo @@ -951,13 +959,18 @@ M 644 :1 COPYING """) def test_keep_empty_commit(self): - params = {b'include_paths': None, b'exclude_paths': None, b'squash_empty_commits': False} - self.assertFiltering(_SAMPLE_EMPTY_COMMIT, params, _SAMPLE_EMPTY_COMMIT) + params = { + b'include_paths': None, + b'exclude_paths': None, + b'squash_empty_commits': False, + } + self.assertFiltering( + _SAMPLE_EMPTY_COMMIT, params, _SAMPLE_EMPTY_COMMIT) def test_squash_unresolved_references(self): params = {b'include_paths': None, b'exclude_paths': None} - self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \ -b"""blob + self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, b"""\ +blob mark :1 data 4 foo @@ -994,15 +1007,20 @@ M 644 :99 data/DATA2 """) def test_keep_unresolved_from_and_merge(self): - params = {b'include_paths': None, b'exclude_paths': None, b'squash_empty_commits': False} - self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, _SAMPLE_FROM_MERGE_COMMIT) + params = { + b'include_paths': None, + b'exclude_paths': None, + b'squash_empty_commits': False, + } + self.assertFiltering( + _SAMPLE_FROM_MERGE_COMMIT, params, _SAMPLE_FROM_MERGE_COMMIT) def test_with_excludes(self): params = {b'include_paths': None, b'exclude_paths': [b'data/DATA'], b'squash_empty_commits': False} - self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \ -b"""blob + self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, b"""\ +blob mark :1 data 4 foo @@ -1037,8 +1055,8 @@ M 644 :99 data/DATA2 params = {b'include_paths': [b'COPYING', b'data/DATA2'], b'exclude_paths': None, b'squash_empty_commits': False} - self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \ -b"""blob + self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, b"""\ +blob mark :1 data 4 foo @@ -1067,15 +1085,14 @@ from :3 merge :4 merge :1001 M 644 :99 data/DATA2 -""" -) +""") def test_with_directory_includes(self): params = {b'include_paths': [b'data/'], b'exclude_paths': None, b'squash_empty_commits': False} - self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, \ -b"""commit refs/heads/master + self.assertFiltering(_SAMPLE_FROM_MERGE_COMMIT, params, b"""\ +commit refs/heads/master mark :3 committer Joe <joe@example.com> 1234567890 +1000 data 6 diff --git a/fastimport/tests/test_helpers.py b/fastimport/tests/test_helpers.py index 3198cea..f695a60 100644 --- a/fastimport/tests/test_helpers.py +++ b/fastimport/tests/test_helpers.py @@ -51,5 +51,6 @@ class TestCommonDirectory(unittest.TestCase): self.assertEqual(c, b'foo/bar/') def test_lots_of_paths(self): - c = helpers.common_directory([b'foo/bar/x', b'foo/bar/y', b'foo/bar/z']) + c = helpers.common_directory( + [b'foo/bar/x', b'foo/bar/y', b'foo/bar/z']) self.assertEqual(c, b'foo/bar/') diff --git a/fastimport/tests/test_info_processor.py b/fastimport/tests/test_info_processor.py index 6904b50..f86fd1f 100644 --- a/fastimport/tests/test_info_processor.py +++ b/fastimport/tests/test_info_processor.py @@ -39,6 +39,7 @@ initial """ + class TestFastImportInfo(TestCase): def test_simple(self): diff --git a/fastimport/tests/test_parser.py b/fastimport/tests/test_parser.py index 5084dc9..cfbe096 100644 --- a/fastimport/tests/test_parser.py +++ b/fastimport/tests/test_parser.py @@ -143,10 +143,14 @@ multi-author test """ _timefunc = time.time + + class TestImportParser(unittest.TestCase): + def setUp(self): self.fake_time = 42.0123 time.time = lambda: self.fake_time + def tearDown(self): time.time = _timefunc del self.fake_time @@ -181,9 +185,12 @@ class TestImportParser(unittest.TestCase): self.assertEqual(b'commit', cmd4.name) self.assertEqual(b'2', cmd4.mark) self.assertEqual(b':2', cmd4.id) - self.assertEqual(b'initial import', cmd4.message) + self.assertEqual( + b'initial import', cmd4.message) - self.assertEqual((b'bugs bunny', b'bugs@bunny.org', self.fake_time, 0), cmd4.committer) + self.assertEqual( + (b'bugs bunny', b'bugs@bunny.org', self.fake_time, 0), + cmd4.committer) # namedtuple attributes self.assertEqual(b'bugs bunny', cmd4.committer.name) self.assertEqual(b'bugs@bunny.org', cmd4.committer.email) @@ -205,7 +212,8 @@ class TestImportParser(unittest.TestCase): self.assertEqual(None, cmd5.mark) self.assertEqual(b'@19', cmd5.id) self.assertEqual(b'second commit', cmd5.message) - self.assertEqual((b'', b'bugs@bunny.org', self.fake_time, 0), cmd5.committer) + self.assertEqual( + (b'', b'bugs@bunny.org', self.fake_time, 0), cmd5.committer) self.assertEqual(None, cmd5.author) self.assertEqual(19, cmd5.lineno) self.assertEqual(b'refs/heads/master', cmd5.ref) @@ -300,7 +308,8 @@ class TestStringParsing(unittest.TestCase): def test_unquote(self): s = br'hello \"sweet\" wo\\r\tld' - self.assertEqual(br'hello "sweet" wo\r' + b'\tld', + self.assertEqual( + br'hello "sweet" wo\r' + b'\tld', parser._unquote_c_string(s)) @@ -312,7 +321,8 @@ class TestPathPairParsing(unittest.TestCase): def test_path_pair_spaces_in_first(self): p = parser.ImportParser("") - self.assertEqual([b'foo bar', b'baz'], + self.assertEqual( + [b'foo bar', b'baz'], p._path_pair(b'"foo bar" baz')) @@ -328,7 +338,8 @@ class TestTagParsing(unittest.TestCase): cmds = list(p.iter_commands()) self.assertEqual(1, len(cmds)) self.assertTrue(isinstance(cmds[0], commands.TagCommand)) - self.assertEqual(cmds[0].tagger, + self.assertEqual( + cmds[0].tagger, (b'Joe Wong', b'joe@example.com', 1234567890.0, -21600)) def test_tagger_no_email_strict(self): |