diff options
| author | Anthon van der Neut <anthon@mnt.org> | 2016-02-27 10:15:02 +0100 |
|---|---|---|
| committer | Anthon van der Neut <anthon@mnt.org> | 2016-02-27 10:15:02 +0100 |
| commit | 2f9cdc98cc0adea615cb2180481c7780eef48f97 (patch) | |
| tree | ff710ad0913ced7c241f0445e95e6eefeb24ae1b | |
| parent | acc0b296466c5364569b1df8402d2cb95ca01915 (diff) | |
| download | ruamel.yaml-2f9cdc98cc0adea615cb2180481c7780eef48f97.tar.gz | |
pep8 compliance, util.load_yaml_guess_indent0.11.2
52 files changed, 858 insertions, 451 deletions
diff --git a/__init__.py b/__init__.py index 6f09536..fa23f4d 100644 --- a/__init__.py +++ b/__init__.py @@ -9,7 +9,7 @@ from __future__ import absolute_import _package_data = dict( full_package_name="ruamel.yaml", - version_info=(0, 11, 1), + version_info=(0, 11, 2), author="Anthon van der Neut", author_email="a.van.der.neut@ruamel.eu", description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order", # NOQA diff --git a/_test/lib/canonical.py b/_test/lib/canonical.py index c273e01..39dc87a 100644 --- a/_test/lib/canonical.py +++ b/_test/lib/canonical.py @@ -5,9 +5,11 @@ from ruamel.yaml.constructor import Constructor from ruamel.yaml.resolver import Resolver from ruamel.yaml.compat import unichr, PY3 + class CanonicalError(ruamel.yaml.YAMLError): pass + class CanonicalScanner: def __init__(self, data): @@ -214,8 +216,8 @@ class CanonicalScanner: else: found = True -class CanonicalParser: +class CanonicalParser: def __init__(self): self.events = [] self.parsed = False @@ -237,7 +239,7 @@ class CanonicalParser: # document: DIRECTIVE? DOCUMENT-START node def parse_document(self): - node = None + # node = None if self.check_token(ruamel.yaml.DirectiveToken): self.get_token(ruamel.yaml.DirectiveToken) self.get_token(ruamel.yaml.DocumentStartToken) @@ -257,7 +259,8 @@ class CanonicalParser: if self.check_token(ruamel.yaml.TagToken): tag = self.get_token_value() if self.check_token(ruamel.yaml.ScalarToken): - self.events.append(ruamel.yaml.ScalarEvent(anchor, tag, (False, False), self.get_token_value(), None, None)) + self.events.append(ruamel.yaml.ScalarEvent(anchor, tag, (False, False), + self.get_token_value(), None, None)) elif self.check_token(ruamel.yaml.FlowSequenceStartToken): self.events.append(ruamel.yaml.SequenceStartEvent(anchor, tag, None, None)) self.parse_sequence() @@ -265,7 +268,8 @@ class CanonicalParser: self.events.append(ruamel.yaml.MappingStartEvent(anchor, tag, None, None)) self.parse_mapping() else: - raise CanonicalError("SCALAR, '[', or '{' is expected, got "+repr(self.tokens[0])) + raise CanonicalError("SCALAR, '[', or '{' is expected, got " + + repr(self.tokens[0])) # sequence: SEQUENCE-START (node (ENTRY node)*)? ENTRY? SEQUENCE-END def parse_sequence(self): @@ -323,8 +327,9 @@ class CanonicalParser: self.parse() return self.events[0] + class CanonicalLoader(CanonicalScanner, CanonicalParser, - Composer, Constructor, Resolver): + Composer, Constructor, Resolver): def __init__(self, stream): if hasattr(stream, 'read'): @@ -337,33 +342,38 @@ class CanonicalLoader(CanonicalScanner, CanonicalParser, ruamel.yaml.CanonicalLoader = CanonicalLoader + def canonical_scan(stream): return ruamel.yaml.scan(stream, Loader=CanonicalLoader) ruamel.yaml.canonical_scan = canonical_scan + def canonical_parse(stream): return ruamel.yaml.parse(stream, Loader=CanonicalLoader) ruamel.yaml.canonical_parse = canonical_parse + def canonical_compose(stream): return ruamel.yaml.compose(stream, Loader=CanonicalLoader) ruamel.yaml.canonical_compose = canonical_compose + def canonical_compose_all(stream): return ruamel.yaml.compose_all(stream, Loader=CanonicalLoader) ruamel.yaml.canonical_compose_all = canonical_compose_all + def canonical_load(stream): return ruamel.yaml.load(stream, Loader=CanonicalLoader) ruamel.yaml.canonical_load = canonical_load + def canonical_load_all(stream): return ruamel.yaml.load_all(stream, Loader=CanonicalLoader) ruamel.yaml.canonical_load_all = canonical_load_all - diff --git a/_test/lib/test_all.py b/_test/lib/test_all.py index faf2cfb..57485fc 100644 --- a/_test/lib/test_all.py +++ b/_test/lib/test_all.py @@ -1,17 +1,17 @@ -import sys +import sys # NOQA import ruamel.yaml import test_appliance + def main(args=None): collections = [] import test_yaml collections.append(test_yaml) - if yaml.__with_libyaml__: + if ruamel.yaml.__with_libyaml__: import test_yaml_ext collections.append(test_yaml_ext) test_appliance.run(collections, args) if __name__ == '__main__': main() - diff --git a/_test/lib/test_appliance.py b/_test/lib/test_appliance.py index 9f1c364..fd957ad 100644 --- a/_test/lib/test_appliance.py +++ b/_test/lib/test_appliance.py @@ -29,6 +29,7 @@ def find_test_functions(collections): functions.append(value) return functions + def find_test_filenames(directory): filenames = {} for filename in os.listdir(directory): @@ -40,6 +41,7 @@ def find_test_filenames(directory): filenames = sorted(filenames.items()) return filenames + def parse_arguments(args): """""" parser = argparse.ArgumentParser(usage=""" run the yaml tests. By default @@ -98,6 +100,7 @@ def parse_arguments(args): include_filenames.extend(os.environ['YAML_TEST_FILENAMES'].split()) return include_functions, include_filenames, verbose, args + def execute(function, filenames, verbose): if PY3: name = function.__name__ @@ -130,6 +133,7 @@ def execute(function, filenames, verbose): sys.stdout.flush() return (name, filenames, kind, info) + def display(results, verbose): if results and not verbose: sys.stdout.write('\n') @@ -177,6 +181,7 @@ def display(results, verbose): ret_val = 2 return ret_val + def run(collections, args=None): test_functions = find_test_functions(collections) test_filenames = find_test_filenames(DATA) @@ -211,4 +216,3 @@ def run(collections, args=None): result = execute(function, [], verbose) results.append(result) return display(results, verbose=verbose) - diff --git a/_test/lib/test_build.py b/_test/lib/test_build.py index 901e8ed..5d19e3a 100644 --- a/_test/lib/test_build.py +++ b/_test/lib/test_build.py @@ -1,10 +1,13 @@ if __name__ == '__main__': - import sys, os, distutils.util + import sys + import os + import distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), + sys.version[0:3])) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) - import test_yaml, test_appliance + import test_yaml + import test_appliance test_appliance.run(test_yaml) - diff --git a/_test/lib/test_build_ext.py b/_test/lib/test_build_ext.py index ff195d5..92d927e 100644 --- a/_test/lib/test_build_ext.py +++ b/_test/lib/test_build_ext.py @@ -1,11 +1,14 @@ if __name__ == '__main__': - import sys, os, distutils.util + import sys + import os + import distutils.util build_lib = 'build/lib' - build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), sys.version[0:3])) + build_lib_ext = os.path.join('build', 'lib.%s-%s' % (distutils.util.get_platform(), + sys.version[0:3])) sys.path.insert(0, build_lib) sys.path.insert(0, build_lib_ext) - import test_yaml_ext, test_appliance + import test_yaml_ext + import test_appliance test_appliance.run(test_yaml_ext) - diff --git a/_test/lib/test_canonical.py b/_test/lib/test_canonical.py index 797dddc..33387cd 100644 --- a/_test/lib/test_canonical.py +++ b/_test/lib/test_canonical.py @@ -1,13 +1,14 @@ -#from __future__ import absolute_import +# from __future__ import absolute_import from __future__ import print_function import ruamel.yaml -import canonical +import canonical # NOQA + def test_canonical_scanner(canonical_filename, verbose=False): with open(canonical_filename, 'rb') as fp0: data = fp0.read() - tokens = list(yaml.canonical_scan(data)) + tokens = list(ruamel.yaml.canonical_scan(data)) assert tokens, tokens if verbose: for token in tokens: @@ -15,10 +16,11 @@ def test_canonical_scanner(canonical_filename, verbose=False): test_canonical_scanner.unittest = ['.canonical'] + def test_canonical_parser(canonical_filename, verbose=False): with open(canonical_filename, 'rb') as fp0: data = fp0.read() - events = list(yaml.canonical_parse(data)) + events = list(ruamel.yaml.canonical_parse(data)) assert events, events if verbose: for event in events: @@ -26,12 +28,13 @@ def test_canonical_parser(canonical_filename, verbose=False): test_canonical_parser.unittest = ['.canonical'] + def test_canonical_error(data_filename, canonical_filename, verbose=False): with open(data_filename, 'rb') as fp0: data = fp0.read() try: - output = list(yaml.canonical_load_all(data)) - except yaml.YAMLError as exc: + output = list(ruamel.yaml.canonical_load_all(data)) # NOQA + except ruamel.yaml.YAMLError as exc: if verbose: print(exc) else: @@ -43,4 +46,3 @@ test_canonical_error.skip = ['.empty'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_constructor.py b/_test/lib/test_constructor.py index 56ead53..fa8d8df 100644 --- a/_test/lib/test_constructor.py +++ b/_test/lib/test_constructor.py @@ -3,29 +3,32 @@ from __future__ import print_function import ruamel.yaml import pprint -from ruamel.yaml.compat import PY2, PY3 +from ruamel.yaml.compat import PY2 import datetime try: set except NameError: - from sets import Set as set + from sets import Set as set # NOQA import ruamel.yaml.tokens + def execute(code): global value exec(code) return value + def _make_objects(): - global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLobject1, YAMLobject2, \ - AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ - NewArgs, NewArgsWithState, Reduce, ReduceWithState, MyInt, MyList, MyDict, \ - FixedOffset, today, execute + global MyLoader, MyDumper, MyTestClass1, MyTestClass2, MyTestClass3, YAMLobject1, \ + YAMLobject2, AnObject, AnInstance, AState, ACustomState, InitArgs, InitArgsWithState, \ + NewArgs, NewArgsWithState, Reduce, ReduceWithState, MyInt, MyList, MyDict, \ + FixedOffset, today, execute - class MyLoader(yaml.Loader): + class MyLoader(ruamel.yaml.Loader): pass - class MyDumper(yaml.Dumper): + + class MyDumper(ruamel.yaml.Dumper): pass class MyTestClass1: @@ -33,6 +36,7 @@ def _make_objects(): self.x = x self.y = y self.z = z + def __eq__(self, other): if isinstance(other, MyTestClass1): return self.__class__, self.__dict__ == other.__class__, other.__dict__ @@ -42,6 +46,7 @@ def _make_objects(): def construct1(constructor, node): mapping = constructor.construct_mapping(node) return MyTestClass1(**mapping) + def represent1(representer, native): return representer.represent_mapping("!tag1", native.__dict__) @@ -52,16 +57,19 @@ def _make_objects(): ruamel.yaml.loader = MyLoader ruamel.yaml.dumper = MyDumper ruamel.yaml.tag = "!tag2" + def from_yaml(cls, constructor, node): x = constructor.construct_yaml_int(node) return cls(x=x) from_yaml = classmethod(from_yaml) + def to_yaml(cls, representer, native): return representer.represent_scalar(cls.yaml_tag, str(native.x)) to_yaml = classmethod(to_yaml) class MyTestClass3(MyTestClass2): ruamel.yaml.tag = "!tag3" + def from_yaml(cls, constructor, node): mapping = constructor.construct_mapping(node) if '=' in mapping: @@ -70,17 +78,20 @@ def _make_objects(): mapping['x'] = x return cls(**mapping) from_yaml = classmethod(from_yaml) + def to_yaml(cls, representer, native): return representer.represent_mapping(cls.yaml_tag, native.__dict__) to_yaml = classmethod(to_yaml) - class YAMLobject1(yaml.YAMLObject): + class YAMLobject1(ruamel.yaml.YAMLObject): ruamel.yaml.loader = MyLoader ruamel.yaml.dumper = MyDumper ruamel.yaml.tag = '!foo' + def __init__(self, my_parameter=None, my_another_parameter=None): self.my_parameter = my_parameter self.my_another_parameter = my_another_parameter + def __eq__(self, other): if isinstance(other, YAMLobject1): return self.__class__, self.__dict__ == other.__class__, other.__dict__ @@ -91,16 +102,20 @@ def _make_objects(): ruamel.yaml.loader = MyLoader ruamel.yaml.dumper = MyDumper ruamel.yaml.tag = '!bar' + def __init__(self, foo=1, bar=2, baz=3): self.foo = foo self.bar = bar self.baz = baz + def __getstate__(self): return {1: self.foo, 2: self.bar, 3: self.baz} + def __setstate__(self, state): self.foo = state[1] self.bar = state[2] self.baz = state[3] + def __eq__(self, other): if isinstance(other, YAMLobject2): return self.__class__, self.__dict__ == other.__class__, other.__dict__ @@ -114,24 +129,28 @@ def _make_objects(): self.bar = bar self.baz = baz return self + def __cmp__(self, other): return cmp((type(self), self.foo, self.bar, self.baz), - (type(other), other.foo, other.bar, other.baz)) + (type(other), other.foo, other.bar, other.baz)) + def __eq__(self, other): return type(self) is type(other) and \ - (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) + (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) class AnInstance: def __init__(self, foo=None, bar=None, baz=None): self.foo = foo self.bar = bar self.baz = baz + def __cmp__(self, other): return cmp((type(self), self.foo, self.bar, self.baz), - (type(other), other.foo, other.bar, other.baz)) + (type(other), other.foo, other.bar, other.baz)) + def __eq__(self, other): return type(self) is type(other) and \ - (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) + (self.foo, self.bar, self.baz) == (other.foo, other.bar, other.baz) class AState(AnInstance): def __getstate__(self): @@ -140,6 +159,7 @@ def _make_objects(): '_bar': self.bar, '_baz': self.baz, } + def __setstate__(self, state): self.foo = state['_foo'] self.bar = state['_bar'] @@ -148,38 +168,42 @@ def _make_objects(): class ACustomState(AnInstance): def __getstate__(self): return (self.foo, self.bar, self.baz) + def __setstate__(self, state): self.foo, self.bar, self.baz = state - #class InitArgs(AnInstance): - # def __getinitargs__(self): - # return (self.foo, self.bar, self.baz) - # def __getstate__(self): - # return {} + # class InitArgs(AnInstance): + # def __getinitargs__(self): + # return (self.foo, self.bar, self.baz) + # def __getstate__(self): + # return {} - #class InitArgsWithState(AnInstance): - # def __getinitargs__(self): - # return (self.foo, self.bar) - # def __getstate__(self): - # return self.baz - # def __setstate__(self, state): - # self.baz = state + # class InitArgsWithState(AnInstance): + # def __getinitargs__(self): + # return (self.foo, self.bar) + # def __getstate__(self): + # return self.baz + # def __setstate__(self, state): + # self.baz = state class NewArgs(AnObject): def __getnewargs__(self): return (self.foo, self.bar, self.baz) + def __getstate__(self): return {} class NewArgsWithState(AnObject): def __getnewargs__(self): return (self.foo, self.bar) + def __getstate__(self): return self.baz + def __setstate__(self, state): self.baz = state - #if PY3 or PY2: + # if PY3 or PY2: InitArgs = NewArgs InitArgsWithState = NewArgsWithState @@ -191,6 +215,7 @@ def _make_objects(): class ReduceWithState(AnObject): def __reduce__(self): return self.__class__, (self.foo, self.bar), self.baz + def __setstate__(self, state): self.baz = state @@ -201,6 +226,7 @@ def _make_objects(): class MyList(list): def __init__(self, n=1): self.extend([None]*n) + def __eq__(self, other): return type(self) is type(other) and list(self) == list(other) @@ -208,6 +234,7 @@ def _make_objects(): def __init__(self, n=1): for k in range(n): self[k] = None + def __eq__(self, other): return type(self) is type(other) and dict(self) == dict(other) @@ -215,10 +242,13 @@ def _make_objects(): def __init__(self, offset, name): self.__offset = datetime.timedelta(minutes=offset) self.__name = name + def utcoffset(self, dt): return self.__offset + def tzname(self, dt): return self.__name + def dst(self, dt): return datetime.timedelta(0) @@ -229,6 +259,7 @@ try: except: from collections import OrderedDict # to get the right name import ... as ordereddict doesn't do that + class ordereddict(OrderedDict): pass @@ -257,13 +288,14 @@ def _serialize_value(data): else: return str(data) + def test_constructor_types(data_filename, code_filename, verbose=False): _make_objects() native1 = None native2 = None try: with open(data_filename, 'rb') as fp0: - native1 = list(yaml.load_all(fp0, Loader=MyLoader)) + native1 = list(ruamel.yaml.load_all(fp0, Loader=MyLoader)) if len(native1) == 1: native1 = native1[0] with open(code_filename, 'rb') as fp0: @@ -289,16 +321,17 @@ def test_constructor_types(data_filename, code_filename, verbose=False): test_constructor_types.unittest = ['.data', '.code'] + def test_roundtrip_data(code_filename, roundtrip_filename, verbose=False): _make_objects() - with open(code_filename, 'rb') as fp0: + with open(code_filename, 'rb') as fp0: value1 = fp0 .read() - native2 = list(yaml.load_all(value1, Loader=MyLoader)) + native2 = list(ruamel.yaml.load_all(value1, Loader=MyLoader)) if len(native2) == 1: native2 = native2[0] try: value2 = ruamel.yaml.dump(native2, Dumper=MyDumper, default_flow_style=False, - allow_unicode=True, encoding='utf-8') + allow_unicode=True, encoding='utf-8') # value2 += x if verbose: print("SERIALIZED NATIVE1:") @@ -315,8 +348,8 @@ test_roundtrip_data.unittest = ['.data', '.roundtrip'] if __name__ == '__main__': - import sys, test_constructor + import sys + import test_constructor # NOQA sys.modules['test_constructor'] = sys.modules['__main__'] import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_emitter.py b/_test/lib/test_emitter.py index 39f0c1f..1158854 100644 --- a/_test/lib/test_emitter.py +++ b/_test/lib/test_emitter.py @@ -3,6 +3,7 @@ from __future__ import print_function import ruamel.yaml as yaml + def _compare_events(events1, events2): assert len(events1) == len(events2), (events1, events2) for event1, event2 in zip(events1, events2): @@ -16,6 +17,7 @@ def _compare_events(events1, events2): assert event1.tag == event2.tag, (event1, event2) assert event1.value == event2.value, (event1, event2) + def test_emitter_on_data(data_filename, canonical_filename, verbose=False): with open(data_filename, 'rb') as fp0: events = list(yaml.parse(fp0)) @@ -28,6 +30,7 @@ def test_emitter_on_data(data_filename, canonical_filename, verbose=False): test_emitter_on_data.unittest = ['.data', '.canonical'] + def test_emitter_on_canonical(canonical_filename, verbose=False): with open(canonical_filename, 'rb') as fp0: events = list(yaml.parse(fp0)) @@ -41,6 +44,7 @@ def test_emitter_on_canonical(canonical_filename, verbose=False): test_emitter_on_canonical.unittest = ['.canonical'] + def test_emitter_styles(data_filename, canonical_filename, verbose=False): for filename in [data_filename, canonical_filename]: with open(filename, 'rb') as fp0: @@ -51,13 +55,13 @@ def test_emitter_styles(data_filename, canonical_filename, verbose=False): for event in events: if isinstance(event, yaml.ScalarEvent): event = yaml.ScalarEvent(event.anchor, event.tag, - event.implicit, event.value, style=style) + event.implicit, event.value, style=style) elif isinstance(event, yaml.SequenceStartEvent): event = yaml.SequenceStartEvent(event.anchor, event.tag, - event.implicit, flow_style=flow_style) + event.implicit, flow_style=flow_style) elif isinstance(event, yaml.MappingStartEvent): event = yaml.MappingStartEvent(event.anchor, event.tag, - event.implicit, flow_style=flow_style) + event.implicit, flow_style=flow_style) styled_events.append(event) output = yaml.emit(styled_events) if verbose: @@ -69,6 +73,7 @@ def test_emitter_styles(data_filename, canonical_filename, verbose=False): test_emitter_styles.unittest = ['.data', '.canonical'] + class EventsLoader(yaml.Loader): def construct_event(self, node): @@ -77,7 +82,8 @@ class EventsLoader(yaml.Loader): else: mapping = self.construct_mapping(node) class_name = str(node.tag[1:])+'Event' - if class_name in ['AliasEvent', 'ScalarEvent', 'SequenceStartEvent', 'MappingStartEvent']: + if class_name in ['AliasEvent', 'ScalarEvent', 'SequenceStartEvent', + 'MappingStartEvent']: mapping.setdefault('anchor', None) if class_name in ['ScalarEvent', 'SequenceStartEvent', 'MappingStartEvent']: mapping.setdefault('tag', None) @@ -91,6 +97,7 @@ class EventsLoader(yaml.Loader): EventsLoader.add_constructor(None, EventsLoader.construct_event) + def test_emitter_events(events_filename, verbose=False): with open(events_filename, 'rb') as fp0: events = list(yaml.load(fp0, Loader=EventsLoader)) @@ -104,4 +111,3 @@ def test_emitter_events(events_filename, verbose=False): if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_errors.py b/_test/lib/test_errors.py index d795c5d..f89392c 100644 --- a/_test/lib/test_errors.py +++ b/_test/lib/test_errors.py @@ -4,6 +4,7 @@ from __future__ import print_function import ruamel.yaml as yaml import test_emitter + def test_loader_error(error_filename, verbose=False): try: with open(error_filename, 'rb') as fp0: @@ -16,6 +17,7 @@ def test_loader_error(error_filename, verbose=False): test_loader_error.unittest = ['.loader-error'] + def test_loader_error_string(error_filename, verbose=False): try: with open(error_filename, 'rb') as fp0: @@ -28,6 +30,7 @@ def test_loader_error_string(error_filename, verbose=False): test_loader_error_string.unittest = ['.loader-error'] + def test_loader_error_single(error_filename, verbose=False): try: with open(error_filename, 'rb') as fp0: @@ -40,6 +43,7 @@ def test_loader_error_single(error_filename, verbose=False): test_loader_error_single.unittest = ['.single-loader-error'] + def test_emitter_error(error_filename, verbose=False): with open(error_filename, 'rb') as fp0: events = list(yaml.load(fp0, @@ -54,12 +58,12 @@ def test_emitter_error(error_filename, verbose=False): test_emitter_error.unittest = ['.emitter-error'] + def test_dumper_error(error_filename, verbose=False): with open(error_filename, 'rb') as fp0: code = fp0.read() try: import yaml - from yaml.compat import StringIO exec(code) except yaml.YAMLError as exc: if verbose: @@ -72,4 +76,3 @@ test_dumper_error.unittest = ['.dumper-error'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_input_output.py b/_test/lib/test_input_output.py index ef09ed5..399a263 100644 --- a/_test/lib/test_input_output.py +++ b/_test/lib/test_input_output.py @@ -2,7 +2,10 @@ from __future__ import absolute_import from __future__ import print_function import ruamel.yaml as yaml -import codecs, tempfile, os, os.path +import codecs +import tempfile +import os +import os.path from ruamel.yaml.compat import PY2, PY3, StringIO, BytesIO if PY2: @@ -28,9 +31,9 @@ if PY3: output = yaml.load(StringIO(data)) assert output == value, (output, value) for input in [data.encode('utf-8'), - codecs.BOM_UTF8+data.encode('utf-8'), - codecs.BOM_UTF16_BE+data.encode('utf-16-be'), - codecs.BOM_UTF16_LE+data.encode('utf-16-le')]: + codecs.BOM_UTF8+data.encode('utf-8'), + codecs.BOM_UTF16_BE+data.encode('utf-16-be'), + codecs.BOM_UTF16_LE+data.encode('utf-16-le')]: if verbose: print("INPUT:", repr(input[:10]), "...") output = yaml.load(input) @@ -45,9 +48,9 @@ else: output = yaml.load(_unicode_open(StringIO(data.encode('utf-8')), 'utf-8')) assert output == value, (output, value) for input in [data, data.encode('utf-8'), - codecs.BOM_UTF8+data.encode('utf-8'), - codecs.BOM_UTF16_BE+data.encode('utf-16-be'), - codecs.BOM_UTF16_LE+data.encode('utf-16-le')]: + codecs.BOM_UTF8+data.encode('utf-8'), + codecs.BOM_UTF16_BE+data.encode('utf-16-be'), + codecs.BOM_UTF16_LE+data.encode('utf-16-le')]: if verbose: print("INPUT:", repr(input[:10]), "...") output = yaml.load(input) @@ -57,14 +60,15 @@ else: test_unicode_input.unittest = ['.unicode'] + def test_unicode_input_errors(unicode_filename, verbose=False): with open(unicode_filename, 'rb') as fp: data = fp.read().decode('utf-8') for input in [data.encode('latin1', 'ignore'), - data.encode('utf-16-be'), data.encode('utf-16-le'), - codecs.BOM_UTF8+data.encode('utf-16-be'), - codecs.BOM_UTF16_BE+data.encode('utf-16-le'), - codecs.BOM_UTF16_LE+data.encode('utf-8')+b'!']: + data.encode('utf-16-be'), data.encode('utf-16-le'), + codecs.BOM_UTF8+data.encode('utf-16-be'), + codecs.BOM_UTF16_BE+data.encode('utf-16-le'), + codecs.BOM_UTF16_LE+data.encode('utf-8')+b'!']: try: yaml.load(input) except yaml.YAMLError as exc: @@ -100,7 +104,8 @@ if PY3: stream = BytesIO() if encoding is None: try: - yaml.dump(value, stream, encoding=encoding, allow_unicode=allow_unicode) + yaml.dump(value, stream, encoding=encoding, + allow_unicode=allow_unicode) except TypeError as exc: if verbose: print(exc) @@ -138,7 +143,8 @@ else: data1 = yaml.dump(value, allow_unicode=allow_unicode) for encoding in [None, 'utf-8', 'utf-16-be', 'utf-16-le']: stream = StringIO() - yaml.dump(value, _unicode_open(stream, 'utf-8'), encoding=encoding, allow_unicode=allow_unicode) + yaml.dump(value, _unicode_open(stream, 'utf-8'), encoding=encoding, + allow_unicode=allow_unicode) data2 = stream.getvalue() data3 = yaml.dump(value, encoding=encoding, allow_unicode=allow_unicode) stream = StringIO() @@ -170,6 +176,7 @@ else: test_unicode_output.unittest = ['.unicode'] + def test_file_output(unicode_filename, verbose=False): with open(unicode_filename, 'rb') as fp: data = fp.read().decode('utf-8') @@ -197,7 +204,7 @@ def test_file_output(unicode_filename, verbose=False): with open(filename, 'rb') as fp0: data2 = fp0.read() with open(filename, 'wb') as stream: - yaml.dump(data, stream, encoding='utf-16-le', + yaml.dump(data, stream, encoding='utf-16-le', allow_unicode=True) with open(filename, 'rb') as fp0: data3 = fp0.read().decode('utf-16-le')[1:].encode('utf-8') @@ -215,6 +222,7 @@ def test_file_output(unicode_filename, verbose=False): test_file_output.unittest = ['.unicode'] + def test_unicode_transfer(unicode_filename, verbose=False): with open(unicode_filename, 'rb') as fp: data = fp.read().decode('utf-8') @@ -241,8 +249,7 @@ def test_unicode_transfer(unicode_filename, verbose=False): input = (u'\ufeff'+input).encode(encoding) output1 = yaml.emit(yaml.parse(input), allow_unicode=True) stream = StringIO() - yaml.emit(yaml.parse(input), _unicode_open(stream, 'utf-8'), - allow_unicode=True) + yaml.emit(yaml.parse(input), _unicode_open(stream, 'utf-8'), allow_unicode=True) output2 = stream.getvalue() if encoding is None: assert isinstance(output1, unicode), (type(output1), encoding) @@ -257,4 +264,3 @@ test_unicode_transfer.unittest = ['.unicode'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_mark.py b/_test/lib/test_mark.py index c8dc83c..56cb52f 100644 --- a/_test/lib/test_mark.py +++ b/_test/lib/test_mark.py @@ -4,6 +4,7 @@ from __future__ import print_function import ruamel.yaml as yaml from ruamel.yaml.compat import text_type, PY3 + def test_marks(marks_filename, verbose=False): with open(marks_filename, 'r' if PY3 else 'rb') as fp0: inputs = fp0.read().split('---\n')[1:] @@ -34,4 +35,3 @@ test_marks.unittest = ['.marks'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_reader.py b/_test/lib/test_reader.py index 25f67b7..ce5de81 100644 --- a/_test/lib/test_reader.py +++ b/_test/lib/test_reader.py @@ -1,12 +1,13 @@ from __future__ import absolute_import from __future__ import print_function -import codecs +import codecs # NOQA import io from ruamel.yaml.compat import PY2 import ruamel.yaml.reader + def _run_reader(data, verbose): try: stream = ruamel.yaml.py.reader.Reader(data) @@ -18,6 +19,7 @@ def _run_reader(data, verbose): else: raise AssertionError("expected an exception") + def test_stream_error(error_filename, verbose=False): with open(error_filename, 'rb') as fp0: _run_reader(fp0, verbose) @@ -45,4 +47,3 @@ test_stream_error.unittest = ['.stream-error'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_recursive.py b/_test/lib/test_recursive.py index 913d525..824ceaa 100644 --- a/_test/lib/test_recursive.py +++ b/_test/lib/test_recursive.py @@ -3,8 +3,8 @@ from __future__ import print_function import ruamel.yaml as yaml -class AnInstance: +class AnInstance: def __init__(self, foo, bar): self.foo = foo self.bar = bar @@ -12,10 +12,11 @@ class AnInstance: def __repr__(self): try: return "%s(foo=%r, bar=%r)" % (self.__class__.__name__, - self.foo, self.bar) + self.foo, self.bar) except RuntimeError: return "%s(foo=..., bar=...)" % self.__class__.__name__ + class AnInstanceWithState(AnInstance): def __getstate__(self): @@ -24,6 +25,7 @@ class AnInstanceWithState(AnInstance): def __setstate__(self, state): self.foo, self.bar = state['attributes'] + def test_recursive(recursive_filename, verbose=False): context = globals().copy() with open(recursive_filename, 'rb') as fp0: @@ -51,4 +53,3 @@ test_recursive.unittest = ['.recursive'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_representer.py b/_test/lib/test_representer.py index 0bcaa2a..6b449a8 100644 --- a/_test/lib/test_representer.py +++ b/_test/lib/test_representer.py @@ -5,6 +5,7 @@ import ruamel.yaml as yaml import test_constructor import pprint + def test_representer_types(code_filename, verbose=False): test_constructor._make_objects() for allow_unicode in [False, True]: @@ -14,7 +15,7 @@ def test_representer_types(code_filename, verbose=False): native2 = None try: output = yaml.dump(native1, Dumper=test_constructor.MyDumper, - allow_unicode=allow_unicode, encoding=encoding) + allow_unicode=allow_unicode, encoding=encoding) native2 = yaml.load(output, Loader=test_constructor.MyLoader) try: if native1 == native2: @@ -43,4 +44,3 @@ test_representer_types.unittest = ['.code'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_resolver.py b/_test/lib/test_resolver.py index b88c91c..64d401d 100644 --- a/_test/lib/test_resolver.py +++ b/_test/lib/test_resolver.py @@ -5,6 +5,7 @@ import ruamel.yaml as yaml import pprint from ruamel.yaml.compat import PY3 + def test_implicit_resolver(data_filename, detect_filename, verbose=False): correct_tag = None node = None @@ -26,27 +27,30 @@ def test_implicit_resolver(data_filename, detect_filename, verbose=False): test_implicit_resolver.unittest = ['.data', '.detect'] + def _make_path_loader_and_dumper(): global MyLoader, MyDumper class MyLoader(yaml.Loader): pass + class MyDumper(yaml.Dumper): pass yaml.add_path_resolver(u'!root', [], - Loader=MyLoader, Dumper=MyDumper) + Loader=MyLoader, Dumper=MyDumper) yaml.add_path_resolver(u'!root/scalar', [], str, - Loader=MyLoader, Dumper=MyDumper) + Loader=MyLoader, Dumper=MyDumper) yaml.add_path_resolver(u'!root/key11/key12/*', ['key11', 'key12'], - Loader=MyLoader, Dumper=MyDumper) + Loader=MyLoader, Dumper=MyDumper) yaml.add_path_resolver(u'!root/key21/1/*', ['key21', 1], - Loader=MyLoader, Dumper=MyDumper) + Loader=MyLoader, Dumper=MyDumper) yaml.add_path_resolver(u'!root/key31/*/*/key14/map', ['key31', None, None, 'key14'], dict, - Loader=MyLoader, Dumper=MyDumper) + Loader=MyLoader, Dumper=MyDumper) return MyLoader, MyDumper + def _convert_node(node): if isinstance(node, yaml.ScalarNode): return (node.tag, node.value) @@ -61,6 +65,7 @@ def _convert_node(node): value.append((_convert_node(key), _convert_node(item))) return (node.tag, value) + def test_path_resolver_loader(data_filename, path_filename, verbose=False): _make_path_loader_and_dumper() with open(data_filename, 'rb') as fp0: @@ -78,6 +83,7 @@ def test_path_resolver_loader(data_filename, path_filename, verbose=False): test_path_resolver_loader.unittest = ['.data', '.path'] + def test_path_resolver_dumper(data_filename, path_filename, verbose=False): _make_path_loader_and_dumper() for filename in [data_filename, path_filename]: @@ -98,4 +104,3 @@ test_path_resolver_dumper.unittest = ['.data', '.path'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_structure.py b/_test/lib/test_structure.py index 4f65926..a23dac8 100644 --- a/_test/lib/test_structure.py +++ b/_test/lib/test_structure.py @@ -2,10 +2,11 @@ from __future__ import absolute_import from __future__ import print_function import ruamel.yaml as yaml -import canonical +import canonical # NOQA import pprint from ruamel.yaml.compat import text_type, PY3 + def _convert_structure(loader): if loader.check_event(yaml.ScalarEvent): event = loader.get_event() @@ -36,6 +37,7 @@ def _convert_structure(loader): loader.get_event() return '?' + def test_structure(data_filename, structure_filename, verbose=False): nodes1 = [] with open(structure_filename, 'r' if PY3 else 'rb') as fp: @@ -45,7 +47,7 @@ def test_structure(data_filename, structure_filename, verbose=False): loader = yaml.Loader(fp) while loader.check_event(): if loader.check_event(yaml.StreamStartEvent, yaml.StreamEndEvent, - yaml.DocumentStartEvent, yaml.DocumentEndEvent): + yaml.DocumentStartEvent, yaml.DocumentEndEvent): loader.get_event() continue nodes1.append(_convert_structure(loader)) @@ -61,6 +63,7 @@ def test_structure(data_filename, structure_filename, verbose=False): test_structure.unittest = ['.data', '.structure'] + def _compare_events(events1, events2, full=False): assert len(events1) == len(events2), (len(events1), len(events2)) for event1, event2 in zip(events1, events2): @@ -73,6 +76,7 @@ def _compare_events(events1, events2, full=False): if isinstance(event1, yaml.ScalarEvent): assert event1.value == event2.value, (event1, event2) + def test_parser(data_filename, canonical_filename, verbose=False): events1 = None events2 = None @@ -91,6 +95,7 @@ def test_parser(data_filename, canonical_filename, verbose=False): test_parser.unittest = ['.data', '.canonical'] + def test_parser_on_canonical(canonical_filename, verbose=False): events1 = None events2 = None @@ -109,6 +114,7 @@ def test_parser_on_canonical(canonical_filename, verbose=False): test_parser_on_canonical.unittest = ['.canonical'] + def _compare_nodes(node1, node2): assert node1.__class__ == node2.__class__, (node1, node2) assert node1.tag == node2.tag, (node1, node2) @@ -123,6 +129,7 @@ def _compare_nodes(node1, node2): for subnode1, subnode2 in zip(item1, item2): _compare_nodes(subnode1, subnode2) + def test_composer(data_filename, canonical_filename, verbose=False): nodes1 = None nodes2 = None @@ -143,38 +150,46 @@ def test_composer(data_filename, canonical_filename, verbose=False): test_composer.unittest = ['.data', '.canonical'] + def _make_loader(): global MyLoader class MyLoader(yaml.Loader): def construct_sequence(self, node): return tuple(yaml.Loader.construct_sequence(self, node)) + def construct_mapping(self, node): pairs = self.construct_pairs(node) pairs.sort(key=(lambda i: text_type(i))) return pairs + def construct_undefined(self, node): return self.construct_scalar(node) MyLoader.add_constructor(u'tag:yaml.org,2002:map', MyLoader.construct_mapping) MyLoader.add_constructor(None, MyLoader.construct_undefined) + def _make_canonical_loader(): global MyCanonicalLoader class MyCanonicalLoader(yaml.CanonicalLoader): def construct_sequence(self, node): return tuple(yaml.CanonicalLoader.construct_sequence(self, node)) + def construct_mapping(self, node): pairs = self.construct_pairs(node) pairs.sort(key=(lambda i: text_type(i))) return pairs + def construct_undefined(self, node): return self.construct_scalar(node) - MyCanonicalLoader.add_constructor(u'tag:yaml.org,2002:map', MyCanonicalLoader.construct_mapping) + MyCanonicalLoader.add_constructor(u'tag:yaml.org,2002:map', + MyCanonicalLoader.construct_mapping) MyCanonicalLoader.add_constructor(None, MyCanonicalLoader.construct_undefined) + def test_constructor(data_filename, canonical_filename, verbose=False): _make_loader() _make_canonical_loader() @@ -198,4 +213,3 @@ test_constructor.unittest = ['.data', '.canonical'] if __name__ == '__main__': import test_appliance test_appliance.run(globals()) - diff --git a/_test/lib/test_yaml.py b/_test/lib/test_yaml.py index 65a4bc6..c650762 100644 --- a/_test/lib/test_yaml.py +++ b/_test/lib/test_yaml.py @@ -1,17 +1,17 @@ # coding: utf-8 -from test_mark import * # NOQA -from test_reader import * # NOQA -from test_canonical import * # NOQA -from test_tokens import * # NOQA -from test_structure import * # NOQA -from test_errors import * # NOQA -from test_resolver import * # NOQA -from test_constructor import * # NOQA -from test_emitter import * # NOQA -from test_representer import * # NOQA -from test_recursive import * # NOQA -from test_input_output import * # NOQA +from test_mark import * # NOQA +from test_reader import * # NOQA +from test_canonical import * # NOQA +from test_tokens import * # NOQA +from test_structure import * # NOQA +from test_errors import * # NOQA +from test_resolver import * # NOQA +from test_constructor import * # NOQA +from test_emitter import * # NOQA +from test_representer import * # NOQA +from test_recursive import * # NOQA +from test_input_output import * # NOQA if __name__ == '__main__': import sys diff --git a/_test/roundtrip.py b/_test/roundtrip.py index a8b0fb7..bd65c5a 100644 --- a/_test/roundtrip.py +++ b/_test/roundtrip.py @@ -22,33 +22,34 @@ def dedent(data): return textwrap.dedent(data) -def round_trip_load(dinp): +def round_trip_load(inp): + dinp = dedent(inp) return ruamel.yaml.load(dinp, ruamel.yaml.RoundTripLoader) def round_trip_dump(data, indent=None): dumper = ruamel.yaml.RoundTripDumper - return ruamel.yaml.dump(data, default_flow_style=False, Dumper=dumper, indent=indent) + return ruamel.yaml.dump(data, default_flow_style=False, Dumper=dumper, + allow_unicode=True, + indent=indent) -def round_trip(inp, outp=None, extra=None, intermediate=None): - dinp = dedent(inp) - if outp is not None: - doutp = dedent(outp) - else: - doutp = dinp +def round_trip(inp, outp=None, extra=None, intermediate=None, indent=None): + if outp is None: + outp = inp + doutp = dedent(outp) if extra is not None: doutp += extra - data = round_trip_load(dinp) + data = round_trip_load(inp) if intermediate is not None: if isinstance(intermediate, dict): for k, v in intermediate.items(): if data[k] != v: print('{0!r} <> {1!r}'.format(data[k], v)) raise ValueError - res = round_trip_dump(data) + res = round_trip_dump(data, indent=indent) print('roundtrip data:\n', res, sep='') assert res == doutp - res = round_trip_dump(data) + res = round_trip_dump(data, indent=indent) print('roundtrip second round data:\n', res, sep='') assert res == doutp diff --git a/_test/test_anchor.py b/_test/test_anchor.py index 1735bdd..929b1ed 100644 --- a/_test/test_anchor.py +++ b/_test/test_anchor.py @@ -74,7 +74,6 @@ class TestAnchorsAliases: assert e.yaml_anchor().value == 'etemplate' assert e.yaml_anchor().always_dump is False - # @pytest.mark.xfail def test_anchor_id_retained(self): data = load(""" a: &id002 diff --git a/_test/test_comments.py b/_test/test_comments.py index 501f06d..81bae93 100644 --- a/_test/test_comments.py +++ b/_test/test_comments.py @@ -72,11 +72,11 @@ class TestComments: """) def test_reindent(self): - x = dedent("""\ + x = """\ a: b: # comment 1 c: 1 # comment 2 - """) + """ d = round_trip_load(x) y = round_trip_dump(d, indent=4) assert y == dedent("""\ @@ -181,7 +181,7 @@ class TestComments: """) def test_substitute(self): - x = dedent(""" + x = """ args: username: anthon # name passwd: secret # password @@ -190,12 +190,12 @@ class TestComments: session-name: test loop: wait: 10 - """) + """ data = round_trip_load(x) data['args']['passwd'] = 'deleted password' # note the requirement to add spaces for alignment of comment x = x.replace(': secret ', ': deleted password') - assert round_trip_dump(data) == x + assert round_trip_dump(data) == dedent(x) def test_set_comment(self): round_trip(""" @@ -208,20 +208,6 @@ class TestComments: # this is the end """) - @pytest.mark.xfail - def XXXtest_set_comment_before_tag(self): - # no comments before tags - round_trip(""" - # the beginning - !!set - # or this one? - ? a - # next one is B (lowercase) - ? b # You see? Promised you. - ? c - # this is the end - """) - def test_omap_comment_roundtrip(self): round_trip(""" !!omap @@ -241,8 +227,7 @@ class TestComments: - d: 4 """) - @pytest.mark.xfail - def test_non_ascii_comment(self): + def test_non_ascii(self): round_trip(""" verbosity: 1 # 0 is minimal output, -1 none base_url: http://gopher.net @@ -252,22 +237,22 @@ class TestComments: - 19 - 32 asia and europe: &asia_europe - Turkey: Ankara - Russia: Moscow + Turkey: Ankara + Russia: Moscow countries: - Asia: - <<: *asia_europe - Japan: Tokyo # 東京 - Europe: - <<: *asia_europe - Spain: Madrid - Italy: Rome + Asia: + <<: *asia_europe + Japan: Tokyo # 東京 + Europe: + <<: *asia_europe + Spain: Madrid + Italy: Rome """) class TestMultiLevelGet: def test_mlget_00(self): - x = dedent("""\ + x = """\ a: - b: c: 42 @@ -275,19 +260,20 @@ class TestMultiLevelGet: f: 196 e: g: 3.14 - """) + """ d = round_trip_load(x) assert d.mlget(['a', 1, 'd', 'f'], list_ok=True) == 196 with pytest.raises(AssertionError): d.mlget(['a', 1, 'd', 'f']) == 196 + class TestInsertPopList: """list insertion is more complex than dict insertion, as you need to move the values to subsequent keys on insert""" @property def ins(self): - return dedent("""\ + return """\ ab: - a # a - b # b @@ -297,7 +283,7 @@ class TestInsertPopList: de: - 1 - 2 - """) + """ def test_insert_0(self): d = round_trip_load(self.ins) @@ -316,7 +302,6 @@ class TestInsertPopList: - 2 """) - def test_insert_1(self): d = round_trip_load(self.ins) d['ab'].insert(4, 'xyz') @@ -334,7 +319,7 @@ class TestInsertPopList: - 2 """) - def test_insert_1(self): + def test_insert_2(self): d = round_trip_load(self.ins) d['ab'].insert(1, 'xyz') y = round_trip_dump(d, indent=2) @@ -351,7 +336,6 @@ class TestInsertPopList: - 2 """) - def test_pop_0(self): d = round_trip_load(self.ins) d['ab'].pop(0) diff --git a/_test/test_fail.py b/_test/test_fail.py new file mode 100644 index 0000000..de5bdb8 --- /dev/null +++ b/_test/test_fail.py @@ -0,0 +1,200 @@ +# coding: utf-8 + +# there is some work to do +# provide a failing test xyz and a non-failing xyz_no_fail ( to see +# what the current failing output is. +# on fix of ruamel.yaml, move the marked test to the appropriate test (without mark) +# and remove remove the xyz_no_fail + +import pytest + +from roundtrip import round_trip, dedent, round_trip_load, round_trip_dump + + +class TestCommentFailures: + @pytest.mark.xfail + def test_set_comment_before_tag(self): + # no comments before tags + round_trip(""" + # the beginning + !!set + # or this one? + ? a + # next one is B (lowercase) + ? b # You see? Promised you. + ? c + # this is the end + """) + + def test_set_comment_before_tag_no_fail(self): + # no comments before tags + assert round_trip_dump(round_trip_load(""" + # the beginning + !!set + # or this one? + ? a + # next one is B (lowercase) + ? b # You see? Promised you. + ? c + # this is the end + """)) == dedent(""" + !!set + # or this one? + ? a + # next one is B (lowercase) + ? b # You see? Promised you. + ? c + # this is the end + """) + + @pytest.mark.xfail + def test_comment_dash_line(self): + round_trip(""" + - # abc + a: 1 + b: 2 + """) + + def test_comment_dash_line_fail(self): + x = """ + - # abc + a: 1 + b: 2 + """ + data = round_trip_load(x) + # this is not nice + assert round_trip_dump(data) == dedent(""" + # abc + - a: 1 + b: 2 + """) + + +class TestIndentFailures: + + @pytest.mark.xfail + def test_roundtrip_four_space_indents(self): + s = ( + 'a:\n' + '- foo\n' + '- bar\n' + ) + output = round_trip(s) + assert s == output + + def test_roundtrip_four_space_indents_no_fail(self): + assert round_trip_dump(round_trip_load(""" + a: + - foo + - bar + """), indent=4) == dedent(""" + a: + - foo + - bar + """) + + @pytest.mark.xfail + def test_indent_not_retained(self): + round_trip(""" + verbosity: 1 # 0 is minimal output, -1 none + base_url: http://gopher.net + special_indices: [1, 5, 8] + also_special: + - a + - 19 + - 32 + asia and europe: &asia_europe + Turkey: Ankara + Russia: Moscow + countries: + Asia: + <<: *asia_europe + Japan: Tokyo # 東京 + Europe: + <<: *asia_europe + Spain: Madrid + Italy: Rome + Antarctica: + - too cold + """) + + def test_indent_not_retained_no_fail(self): + assert round_trip_dump(round_trip_load(""" + verbosity: 1 # 0 is minimal output, -1 none + base_url: http://gopher.net + special_indices: [1, 5, 8] + also_special: + - a + - 19 + - 32 + asia and europe: &asia_europe + Turkey: Ankara + Russia: Moscow + countries: + Asia: + <<: *asia_europe + Japan: Tokyo # 東京 + Europe: + <<: *asia_europe + Spain: Madrid + Italy: Rome + Antarctica: + - too cold + """), indent=4) == dedent(""" + verbosity: 1 # 0 is minimal output, -1 none + base_url: http://gopher.net + special_indices: [1, 5, 8] + also_special: + - a + - 19 + - 32 + asia and europe: &asia_europe + Turkey: Ankara + Russia: Moscow + countries: + Asia: + <<: *asia_europe + Japan: Tokyo # 東京 + Europe: + <<: *asia_europe + Spain: Madrid + Italy: Rome + Antarctica: + - too cold + """) + + @pytest.mark.xfail + def test_indent_top_level(self): + round_trip(""" + - a: + - b + """, indent=4) + + def test_indent_top_level_no_fail(self): + round_trip(""" + - a: + - b + """, indent=4) + + +class TestTagFailures: + @pytest.mark.xfail + def test_standard_short_tag(self): + round_trip("""\ + !!map + name: Anthon + location: Germany + language: python + """) + + def test_standard_short_tag_no_fail(self): + assert round_trip_dump(round_trip_load(""" + !!map + name: Anthon + location: Germany + language: python + """)) == dedent(""" + name: Anthon + location: Germany + language: python + """) diff --git a/_test/test_indentation.py b/_test/test_indentation.py index 9d5fc96..c6131f1 100644 --- a/_test/test_indentation.py +++ b/_test/test_indentation.py @@ -19,89 +19,73 @@ def rt(s): ).strip() + '\n' -def test_roundtrip_inline_list(): - s = 'a: [a, b, c]\n' - output = rt(s) - assert s == output - - -def test_roundtrip_mapping_of_inline_lists(): - s = dedent("""\ - a: [a, b, c] - j: [k, l, m] - """) - output = rt(s) - assert s == output - - -def test_roundtrip_mapping_of_inline_lists_comments(): - s = dedent("""\ - # comment A - a: [a, b, c] - # comment B - j: [k, l, m] - """) - output = rt(s) - assert s == output - - -def test_roundtrip_mapping_of_inline_sequence_eol_comments(): - s = dedent("""\ - # comment A - a: [a, b, c] # comment B - j: [k, l, m] # comment C - """) - output = rt(s) - assert s == output - - -# first test by explicitly setting flow style -def test_added_inline_list(): - s1 = dedent(""" - a: - - b - - c - - d - """) - s = 'a: [b, c, d]\n' - data = ruamel.yaml.load(s1, Loader=ruamel.yaml.RoundTripLoader) - val = data['a'] - val.fa.set_flow_style() - # print(type(val), '_yaml_format' in dir(val)) - output = ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper) - assert s == output - -# ############ flow mappings - - -def test_roundtrip_flow_mapping(): - s = dedent("""\ - - {a: 1, b: hallo} - - {j: fka, k: 42} - """) - data = ruamel.yaml.load(s, Loader=ruamel.yaml.RoundTripLoader) - output = ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper) - assert s == output - - -def test_roundtrip_sequence_of_inline_mappings_eol_comments(): - s = dedent("""\ - # comment A - - {a: 1, b: hallo} # comment B - - {j: fka, k: 42} # comment C - """) - output = rt(s) - assert s == output +class TestIndent: + def test_roundtrip_inline_list(self): + s = 'a: [a, b, c]\n' + output = rt(s) + assert s == output + + def test_roundtrip_mapping_of_inline_lists(self): + s = dedent("""\ + a: [a, b, c] + j: [k, l, m] + """) + output = rt(s) + assert s == output + + def test_roundtrip_mapping_of_inline_lists_comments(self): + s = dedent("""\ + # comment A + a: [a, b, c] + # comment B + j: [k, l, m] + """) + output = rt(s) + assert s == output + + def test_roundtrip_mapping_of_inline_sequence_eol_comments(self): + s = dedent("""\ + # comment A + a: [a, b, c] # comment B + j: [k, l, m] # comment C + """) + output = rt(s) + assert s == output + + # first test by explicitly setting flow style + def test_added_inline_list(self): + s1 = dedent(""" + a: + - b + - c + - d + """) + s = 'a: [b, c, d]\n' + data = ruamel.yaml.load(s1, Loader=ruamel.yaml.RoundTripLoader) + val = data['a'] + val.fa.set_flow_style() + # print(type(val), '_yaml_format' in dir(val)) + output = ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper) + assert s == output + + # ############ flow mappings + + def test_roundtrip_flow_mapping(self): + s = dedent("""\ + - {a: 1, b: hallo} + - {j: fka, k: 42} + """) + data = ruamel.yaml.load(s, Loader=ruamel.yaml.RoundTripLoader) + output = ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper) + assert s == output + + def test_roundtrip_sequence_of_inline_mappings_eol_comments(self): + s = dedent("""\ + # comment A + - {a: 1, b: hallo} # comment B + - {j: fka, k: 42} # comment C + """) + output = rt(s) + assert s == output # ############ indentation - - -@pytest.mark.xfail -def test_roundtrip_four_space_indents(): - s = ( - 'a:\n' - '- foo\n' - '- bar\n' - ) - output = rt(s) - assert s == output diff --git a/_test/test_tag.py b/_test/test_tag.py new file mode 100644 index 0000000..872f96a --- /dev/null +++ b/_test/test_tag.py @@ -0,0 +1,32 @@ +# coding: utf-8 + +import pytest # NOQA + +from roundtrip import round_trip + + +class TestIndentFailures: + + def test_tag(self): + round_trip("""\ + !!python/object:__main__.Developer + name: Anthon + location: Germany + language: python + """) + + def test_full_tag(self): + round_trip("""\ + !!tag:yaml.org,2002:python/object:__main__.Developer + name: Anthon + location: Germany + language: python + """) + + def test_standard_tag(self): + round_trip("""\ + !!tag:yaml.org,2002:python/object:map + name: Anthon + location: Germany + language: python + """) diff --git a/_test/test_version.py b/_test/test_version.py index 55d80c3..cadcf44 100644 --- a/_test/test_version.py +++ b/_test/test_version.py @@ -5,11 +5,11 @@ import pytest # NOQA import ruamel.yaml from roundtrip import dedent + def load(s, version=None): return ruamel.yaml.round_trip_load(dedent(s), version) - class TestVersions: def test_explicit_1_2(self): l = load("""\ @@ -27,7 +27,7 @@ class TestVersions: """) assert l[0] == '12:34:56' assert l[1] == 12 - assert l[2] == '012345678' + assert l[2] == '012345678' assert l[3] == 10 assert l[4] == 'on' assert l[5] == 'off' @@ -51,7 +51,7 @@ class TestVersions: """) assert l[0] == 45296 assert l[1] == 10 - assert l[2] == '012345678' + assert l[2] == '012345678' assert l[3] == 10 assert l[4] is True assert l[5] is False @@ -73,7 +73,7 @@ class TestVersions: """) assert l[0] == '12:34:56' assert l[1] == 12 - assert l[2] == '012345678' + assert l[2] == '012345678' assert l[3] == 10 assert l[4] == 'on' assert l[5] == 'off' @@ -95,11 +95,10 @@ class TestVersions: """, version="1.1") assert l[0] == 45296 assert l[1] == 10 - assert l[2] == '012345678' + assert l[2] == '012345678' assert l[3] == 10 assert l[4] is True assert l[5] is False assert l[6] is True assert l[7] is False assert l[8] is True - diff --git a/_test/test_z_data.py b/_test/test_z_data.py index 2145a19..3b72618 100644 --- a/_test/test_z_data.py +++ b/_test/test_z_data.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import print_function @@ -8,8 +9,8 @@ import platform # NOQA sys.path.insert(0, os.path.dirname(__file__) + '/lib') -import ruamel.yaml -import test_appliance +import ruamel.yaml # NOQA +import test_appliance # NOQA args = [] diff --git a/comments.py b/comments.py index 7cac4f6..8e5ebb8 100644 --- a/comments.py +++ b/comments.py @@ -3,9 +3,6 @@ from __future__ import absolute_import from __future__ import print_function -__all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap", - "CommentedSet", 'comment_attrib', 'merge_attrib'] - """ stuff to deal with comments and formatting on dict/list/ordereddict/set these are not really related, formatting could be factored out as @@ -14,6 +11,10 @@ a separate base from collections import MutableSet +__all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap", + "CommentedSet", 'comment_attrib', 'merge_attrib'] + + try: from .compat import ordereddict except ImportError: @@ -24,6 +25,7 @@ format_attrib = '_yaml_format' line_col_attrib = '_yaml_line_col' anchor_attrib = '_yaml_anchor' merge_attrib = '_yaml_merge' +tag_attrib = '_yaml_tag' class Comment(object): @@ -140,6 +142,14 @@ class Anchor(object): self.always_dump = False +class Tag(object): + """store tag information for roundtripping""" + attrib = tag_attrib + + def __init__(self): + self.value = None + + class CommentedBase(object): @property def ca(self): @@ -245,6 +255,15 @@ class CommentedBase(object): self.anchor.value = value self.anchor.always_dump = always_dump + @property + def tag(self): + if not hasattr(self, Tag.attrib): + setattr(self, Tag.attrib, Tag()) + return getattr(self, Tag.attrib) + + def yaml_set_tag(self, value): + self.tag.value = value + class CommentedSeq(list, CommentedBase): __slots__ = [Comment.attrib, ] @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import print_function diff --git a/composer.py b/composer.py index 6a7e439..cf2508e 100644 --- a/composer.py +++ b/composer.py @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import from __future__ import print_function diff --git a/configobjwalker.py b/configobjwalker.py index 1fe6f35..bab910c 100644 --- a/configobjwalker.py +++ b/configobjwalker.py @@ -1,65 +1,9 @@ +# coding: utf-8 - -def configobj_walker(cfg): - """ - walks over a ConfigObj (INI file with comments) generating - corresponding YAML output (including comments - """ - from configobj import ConfigObj - assert isinstance(cfg, ConfigObj) - for c in cfg.initial_comment: - if c.strip(): - yield c - for s in _walk_section(cfg): - if s.strip(): - yield s - for c in cfg.final_comment: - if c.strip(): - yield c +import warnings +from ruamel.yaml.util import configobj_walker as new_configobj_walker -def _walk_section(s, level=0): - from configobj import Section - assert isinstance(s, Section) - indent = u' ' * level - for name in s.scalars: - for c in s.comments[name]: - yield indent + c.strip() - x = s[name] - if u'\n' in x: - i = indent + u' ' - x = u'|\n' + i + x.strip().replace(u'\n', u'\n' + i) - elif ':' in x: - x = u"'" + x.replace(u"'", u"''") + u"'" - line = u'{0}{1}: {2}'.format(indent, name, x) - c = s.inline_comments[name] - if c: - line += u' ' + c - yield line - for name in s.sections: - for c in s.comments[name]: - yield indent + c.strip() - line = u'{0}{1}:'.format(indent, name) - c = s.inline_comments[name] - if c: - line += u' ' + c - yield line - for val in _walk_section(s[name], level=level+1): - yield val - -# def config_obj_2_rt_yaml(cfg): -# from .comments import CommentedMap, CommentedSeq -# from configobj import ConfigObj -# assert isinstance(cfg, ConfigObj) -# #for c in cfg.initial_comment: -# # if c.strip(): -# # pass -# cm = CommentedMap() -# for name in s.sections: -# cm[name] = d = CommentedMap() -# -# -# #for c in cfg.final_comment: -# # if c.strip(): -# # yield c -# return cm +def configobj_walker(cfg): + warnings.warn("configobj_walker has move to ruamel.yaml.util, please update your code") + return new_configobj_walker(cfg) diff --git a/constructor.py b/constructor.py index 9ae2e23..81635de 100644 --- a/constructor.py +++ b/constructor.py @@ -1,9 +1,8 @@ +# coding: utf-8 + from __future__ import absolute_import from __future__ import print_function -__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor', - 'ConstructorError', 'RoundTripConstructor'] - import collections import datetime import base64 @@ -15,8 +14,7 @@ import types try: from .error import * # NOQA from .nodes import * # NOQA - from .compat import (utf8, builtins_module, to_str, PY2, PY3, ordereddict, - text_type) + from .compat import utf8, builtins_module, to_str, PY2, PY3, ordereddict, text_type from .comments import * # NOQA from .scalarstring import * # NOQA except (ImportError, ValueError): # for Jython @@ -28,6 +26,10 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.scalarstring import * # NOQA +__all__ = ['BaseConstructor', 'SafeConstructor', 'Constructor', + 'ConstructorError', 'RoundTripConstructor'] + + class ConstructorError(MarkedYAMLError): pass @@ -935,10 +937,9 @@ class RoundTripConstructor(SafeConstructor): None, None, "expected a mapping node, but found %s" % node.id, node.start_mark) - if isinstance(node, MappingNode): - merge_map = self.flatten_mapping(node) - if merge_map: - maptyp.add_yaml_merge(merge_map) + merge_map = self.flatten_mapping(node) + if merge_map: + maptyp.add_yaml_merge(merge_map) # mapping = {} if node.comment: maptyp._yaml_add_comment(node.comment[:2]) @@ -1089,6 +1090,25 @@ class RoundTripConstructor(SafeConstructor): yield data self.construct_setting(node, data) + def construct_undefined(self, node): + try: + data = CommentedMap() + data._yaml_set_line_col(node.start_mark.line, node.start_mark.column) + if node.flow_style is True: + data.fa.set_flow_style() + elif node.flow_style is False: + data.fa.set_block_style() + data.yaml_set_tag(node.tag) + yield data + self.construct_mapping(node, data) + except: + raise ConstructorError( + None, None, + "could not determine a constructor for the tag %r" % + utf8(node.tag), + node.start_mark) + + RoundTripConstructor.add_constructor( u'tag:yaml.org,2002:null', RoundTripConstructor.construct_yaml_null) @@ -1,7 +1,6 @@ -from __future__ import absolute_import +# coding: utf-8 -__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader', - 'CBaseDumper', 'CSafeDumper', 'CDumper'] +from __future__ import absolute_import from _ruamel_yaml import CParser, CEmitter @@ -16,6 +15,9 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.representer import * # NOQA from ruamel.yaml.resolver import * # NOQA +__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper'] + class CBaseLoader(CParser, BaseConstructor, BaseResolver): def __init__(self, stream, version=None): @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import __all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'RoundTripDumper'] @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import from __future__ import print_function @@ -271,8 +273,8 @@ class Emitter(object): if self.event.flow_style is False and self.event.comment: self.write_post_comment(self.event) # print('seq event', self.event) - if self.flow_level or self.canonical or self.event.flow_style \ - or self.check_empty_sequence(): + if self.flow_level or self.canonical or self.event.flow_style or \ + self.check_empty_sequence(): self.expect_flow_sequence() else: self.expect_block_sequence() @@ -478,20 +480,19 @@ class Emitter(object): # Checkers. def check_empty_sequence(self): - return (isinstance(self.event, SequenceStartEvent) and self.events - and isinstance(self.events[0], SequenceEndEvent)) + return (isinstance(self.event, SequenceStartEvent) and self.events and + isinstance(self.events[0], SequenceEndEvent)) def check_empty_mapping(self): - return (isinstance(self.event, MappingStartEvent) and self.events - and isinstance(self.events[0], MappingEndEvent)) + return (isinstance(self.event, MappingStartEvent) and self.events and + isinstance(self.events[0], MappingEndEvent)) def check_empty_document(self): if not isinstance(self.event, DocumentStartEvent) or not self.events: return False event = self.events[0] - return (isinstance(event, ScalarEvent) and event.anchor is None - and event.tag is None and event.implicit and - event.value == u'') + return (isinstance(event, ScalarEvent) and event.anchor is None and + event.tag is None and event.implicit and event.value == u'') def check_simple_key(self): length = 0 @@ -509,10 +510,10 @@ class Emitter(object): self.analysis = self.analyze_scalar(self.event.value) length += len(self.analysis.scalar) return (length < self.MAX_SIMPLE_KEY_LENGTH and ( - isinstance(self.event, AliasEvent) - or (isinstance(self.event, ScalarEvent) - and not self.analysis.empty and not self.analysis.multiline) - or self.check_empty_sequence() or self.check_empty_mapping())) + isinstance(self.event, AliasEvent) or + (isinstance(self.event, ScalarEvent) and + not self.analysis.empty and not self.analysis.multiline) or + self.check_empty_sequence() or self.check_empty_mapping())) # Anchor, Tag, and Scalar processors. @@ -532,8 +533,8 @@ class Emitter(object): if self.style is None: self.style = self.choose_scalar_style() if ((not self.canonical or tag is None) and - ((self.style == '' and self.event.implicit[0]) - or (self.style != '' and self.event.implicit[1]))): + ((self.style == '' and self.event.implicit[0]) or + (self.style != '' and self.event.implicit[1]))): self.prepared_tag = None return if self.event.implicit[0] and tag is None: @@ -559,14 +560,13 @@ class Emitter(object): if (not self.event.style or self.event.style == '?') and \ self.event.implicit[0]: if (not (self.simple_key_context and - (self.analysis.empty or self.analysis.multiline)) - and (self.flow_level and self.analysis.allow_flow_plain - or (not self.flow_level and - self.analysis.allow_block_plain))): + (self.analysis.empty or self.analysis.multiline)) and + (self.flow_level and self.analysis.allow_flow_plain or + (not self.flow_level and self.analysis.allow_block_plain))): return '' if self.event.style and self.event.style in '|>': - if (not self.flow_level and not self.simple_key_context - and self.analysis.allow_block): + if (not self.flow_level and not self.simple_key_context and + self.analysis.allow_block): return self.event.style if not self.event.style or self.event.style == '\'': if (self.analysis.allow_single_quoted and @@ -764,8 +764,8 @@ class Emitter(object): if ch in u'\n\x85\u2028\u2029': line_breaks = True if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): - if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' - or u'\uE000' <= ch <= u'\uFFFD') and ch != u'\uFEFF': + if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' or + u'\uE000' <= ch <= u'\uFFFD') and ch != u'\uFEFF': # unicode_characters = True if not self.allow_unicode: special_characters = True @@ -810,8 +810,7 @@ class Emitter(object): allow_block = True # Leading and trailing whitespaces are bad for plain scalars. - if (leading_space or leading_break - or trailing_space or trailing_break): + if (leading_space or leading_break or trailing_space or trailing_break): allow_flow_plain = allow_block_plain = False # We do not permit trailing spaces for block scalars. @@ -998,10 +997,9 @@ class Emitter(object): if end < len(text): ch = text[end] if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \ - or not (u'\x20' <= ch <= u'\x7E' - or (self.allow_unicode - and (u'\xA0' <= ch <= u'\uD7FF' - or u'\uE000' <= ch <= u'\uFFFD'))): + or not (u'\x20' <= ch <= u'\x7E' or + (self.allow_unicode and + (u'\xA0' <= ch <= u'\uD7FF' or u'\uE000' <= ch <= u'\uFFFD'))): if start < end: data = text[start:end] self.column += len(data) @@ -1206,6 +1204,7 @@ class Emitter(object): def write_comment(self, comment): value = comment.value + print('################## comment', repr(value)) # print('{:02d} {:02d} {}'.format(self.column, comment.start_mark.column, value)) if value[-1] == '\n': value = value[:-1] @@ -1231,7 +1230,6 @@ class Emitter(object): pass self.stream.write(value) except TypeError: - print('TypeError while trying to write', repr(value), type(value)) raise self.write_line_break() @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import __all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] @@ -69,10 +71,10 @@ class MarkedYAMLError(YAMLError): if self.context is not None: lines.append(self.context) if self.context_mark is not None \ - and (self.problem is None or self.problem_mark is None - or self.context_mark.name != self.problem_mark.name - or self.context_mark.line != self.problem_mark.line - or self.context_mark.column != self.problem_mark.column): + and (self.problem is None or self.problem_mark is None or + self.context_mark.name != self.problem_mark.name or + self.context_mark.line != self.problem_mark.line or + self.context_mark.column != self.problem_mark.column): lines.append(str(self.context_mark)) if self.problem is not None: lines.append(self.problem) @@ -1,3 +1,4 @@ +# coding: utf-8 # Abstract classes. diff --git a/example/anchor_merge.py b/example/anchor_merge.py index caeddba..3db92d9 100644 --- a/example/anchor_merge.py +++ b/example/anchor_merge.py @@ -26,4 +26,3 @@ inp = """\ data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader) assert data[7]['y'] == 2 - diff --git a/example/so_13517753.py b/example/so_13517753.py index b83af86..af1fc2a 100644 --- a/example/so_13517753.py +++ b/example/so_13517753.py @@ -4,25 +4,26 @@ from __future__ import print_function import ruamel.yaml from ruamel.yaml.comments import CommentedMap -##class MyObj(): -## name = "boby" -## age = 34 -## -##print(ruamel.yaml.dump(MyObj())) # , Dumper=ruamel.yaml.RoundTripDumper), end='') -## -##inp = """\ -##boby: # this is the name -## age: 34 # in years -##""" -## -##print('====', ruamel.yaml.load(inp)) -## -## -##data1 = ruamel.yaml.load(inp, Loader=ruamel.yaml.RoundTripLoader) -##print('<<<', data1.ca.items) -##print(ruamel.yaml.dump(data1, Dumper=ruamel.yaml.RoundTripDumper), end='') -## -##print('----------------') +# class MyObj(): +# name = "boby" +# age = 34 +# +# print(ruamel.yaml.dump(MyObj())) # , Dumper=ruamel.yaml.RoundTripDumper), end='') +# +# inp = """\ +# boby: # this is the name +# age: 34 # in years +# """ +# +# print('====', ruamel.yaml.load(inp)) +# +# +# data1 = ruamel.yaml.load(inp, Loader=ruamel.yaml.RoundTripLoader) +# print('<<<', data1.ca.items) +# print(ruamel.yaml.dump(data1, Dumper=ruamel.yaml.RoundTripDumper), end='') +# +# print('----------------') + class MyObj(): name = "boby" @@ -48,7 +49,4 @@ ruamel.yaml.RoundTripDumper.add_representer(MyObj, MyObj.yaml_representer) data = MyObj() - print(ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper), end='') - - @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import __all__ = ['BaseLoader', 'SafeLoader', 'Loader', 'RoundTripLoader'] @@ -48,12 +50,12 @@ class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): Resolver.__init__(self) -class RoundTripLoader(Reader, RoundTripScanner, Parser, Composer, +class RoundTripLoader(Reader, RoundTripScanner, RoundTripParser, Composer, RoundTripConstructor, VersionedResolver): def __init__(self, stream, version=None): Reader.__init__(self, stream) RoundTripScanner.__init__(self) - Parser.__init__(self) + RoundTripParser.__init__(self) Composer.__init__(self) RoundTripConstructor.__init__(self) VersionedResolver.__init__(self, version) @@ -107,6 +107,7 @@ def safe_load_all(stream, version=None): """ return load_all(stream, SafeLoader, version) + def round_trip_load(stream, version=None): """ Parse the first YAML document in a stream diff --git a/make_win_whl.py b/make_win_whl.py deleted file mode 100644 index df6a2ae..0000000 --- a/make_win_whl.py +++ /dev/null @@ -1,34 +0,0 @@ - -from __future__ import print_function - -""" -The windows whl file has no C stuff but a -ruamel.yaml-0.9.2-py2-none-any.whl file overrules the .tar.gz on Linux. - -You can create a .whl and copy it to impure names (or start -with an impure one), not sure if this is necessary. - -""" - -import sys -import os -import shutil - - -def main(): - src = sys.argv[1] - print(src, '-->') - dir_name = os.path.dirname(src) - base_name = os.path.basename(src) - p, v, rest = base_name.split('-', 2) - # print dir_name - for pyver in ['cp26', 'cp27', 'cp33', 'cp34']: - for platform in ['win32', 'win_amd64']: - dst = os.path.join(dir_name, - '%s-%s-%s-none-%s.whl' % ( - p, v, pyver, platform - )) - print(dst) - shutil.copy(src, dst) - -main() @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import print_function @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import # The following YAML grammar is LL(1) and is parsed by a recursive descent @@ -69,7 +71,7 @@ from __future__ import absolute_import # flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START # FLOW-MAPPING-START KEY } -__all__ = ['Parser', 'ParserError'] +__all__ = ['Parser', 'RoundTripParser', 'ParserError'] try: from .error import MarkedYAMLError @@ -297,6 +299,9 @@ class Parser(object): def parse_block_node_or_indentless_sequence(self): return self.parse_node(block=True, indentless_sequence=True) + def transform_tag(self, handle, suffix): + return self.tag_handles[handle] + suffix + def parse_node(self, block=False, indentless_sequence=False): if self.check_token(AliasToken): token = self.get_token() @@ -333,7 +338,7 @@ class Parser(object): "while parsing a node", start_mark, "found undefined tag handle %r" % utf8(handle), tag_mark) - tag = self.tag_handles[handle]+suffix + tag = self.transform_tag(handle, suffix) else: tag = suffix # if tag == u'!': @@ -661,3 +666,14 @@ class Parser(object): def process_empty_scalar(self, mark): return ScalarEvent(None, None, (True, False), u'', mark, mark) + + +class RoundTripParser(Parser): + """roundtrip is a safe loader, that wants to see the unmangled tag""" + def transform_tag(self, handle, suffix): + # return self.tag_handles[handle]+suffix + if handle == '!!' and suffix in (u'null', u'bool', u'int', u'float', u'binary', + u'timestamp', u'omap', u'pairs', u'set', u'str', + u'seq', u'map'): + return Parser.transform_tag(self, handle, suffix) + return handle+suffix @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import # This module contains abstractions for the input stream. You don't have to # looks further, there are no pretty code. @@ -18,8 +20,6 @@ from __future__ import absolute_import # reader.line, stream.column - the line and the column of the current # character. -__all__ = ['Reader', 'ReaderError'] - import codecs import re @@ -30,6 +30,8 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.error import YAMLError, Mark from ruamel.yaml.compat import text_type, binary_type, PY3 +__all__ = ['Reader', 'ReaderError'] + class ReaderError(YAMLError): diff --git a/representer.py b/representer.py index 22cf720..9d471a2 100644 --- a/representer.py +++ b/representer.py @@ -1,9 +1,8 @@ +# coding: utf-8 + from __future__ import absolute_import from __future__ import print_function -__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', - 'RepresenterError', 'RoundTripRepresenter'] - try: from .error import * # NOQA from .nodes import * # NOQA @@ -26,6 +25,10 @@ else: import copy_reg as copyreg +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError', 'RoundTripRepresenter'] + + class RepresenterError(YAMLError): pass @@ -818,6 +821,17 @@ class RoundTripRepresenter(SafeRepresenter): best_style = best_style return node + def represent_dict(self, data): + """write out tag if safed on loading""" + t = data.tag.value + if t: + while t and t[0] == '!': + t = t[1:] + tag = 'tag:yaml.org,2002:' + t + else: + tag = u'tag:yaml.org,2002:map' + return self.represent_mapping(tag, data) + RoundTripRepresenter.add_representer(type(None), RoundTripRepresenter.represent_none) diff --git a/resolver.py b/resolver.py index 4062602..9ef413d 100644 --- a/resolver.py +++ b/resolver.py @@ -1,6 +1,8 @@ +# coding: utf-8 + from __future__ import absolute_import -__all__ = ['BaseResolver', 'Resolver', 'VersionedResolver'] +import re try: from .error import * # NOQA @@ -11,12 +13,12 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.nodes import * # NOQA from ruamel.yaml.compat import string_types - -import re +__all__ = ['BaseResolver', 'Resolver', 'VersionedResolver'] _DEFAULT_VERSION = (1, 2) + class ResolverError(YAMLError): pass @@ -144,8 +146,8 @@ class BaseResolver(object): and current_index is None: return if isinstance(index_check, string_types): - if not (isinstance(current_index, ScalarNode) - and index_check == current_index.value): + if not (isinstance(current_index, ScalarNode) and + index_check == current_index.value): return elif isinstance(index_check, int) and not isinstance(index_check, bool): @@ -181,6 +183,7 @@ class BaseResolver(object): def processing_version(self): return None + class Resolver(BaseResolver): pass diff --git a/scalarstring.py b/scalarstring.py index 48636f4..3885425 100644 --- a/scalarstring.py +++ b/scalarstring.py @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import from __future__ import print_function @@ -1,3 +1,5 @@ +# coding: utf-8 + from __future__ import absolute_import from __future__ import print_function @@ -759,9 +761,9 @@ class Scanner(object): # '-' character) because we want the flow context to be space # independent. ch = self.peek() - return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ - or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029' - and (ch == u'-' or (not self.flow_level and ch in u'?:'))) + return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' or \ + (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029' and + (ch == u'-' or (not self.flow_level and ch in u'?:'))) # Scanners. diff --git a/serializer.py b/serializer.py index b461ca6..317cf4e 100644 --- a/serializer.py +++ b/serializer.py @@ -1,6 +1,6 @@ -from __future__ import absolute_import +# coding: utf-8 -__all__ = ['Serializer', 'SerializerError'] +from __future__ import absolute_import import re @@ -15,6 +15,8 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.nodes import * # NOQA from ruamel.yaml.compat import nprint, DBG_NODE, dbg +__all__ = ['Serializer', 'SerializerError'] + class SerializerError(YAMLError): pass @@ -114,17 +116,13 @@ class Serializer(object): self.serialized_nodes[node] = True self.descend_resolver(parent, index) if isinstance(node, ScalarNode): - detected_tag = self.resolve(ScalarNode, node.value, - (True, False)) - default_tag = self.resolve(ScalarNode, node.value, - (False, True)) - implicit = \ - (node.tag == detected_tag), (node.tag == default_tag) + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) self.emit(ScalarEvent(alias, node.tag, implicit, node.value, style=node.style, comment=node.comment)) elif isinstance(node, SequenceNode): - implicit = (node.tag - == self.resolve(SequenceNode, node.value, True)) + implicit = (node.tag == self.resolve(SequenceNode, node.value, True)) comment = node.comment # print('comment >>>>>>>>>>>>>.', comment, node.flow_style) end_comment = None @@ -146,8 +144,7 @@ class Serializer(object): index += 1 self.emit(SequenceEndEvent(comment=[seq_comment, end_comment])) elif isinstance(node, MappingNode): - implicit = (node.tag - == self.resolve(MappingNode, node.value, True)) + implicit = (node.tag == self.resolve(MappingNode, node.value, True)) comment = node.comment end_comment = None map_comment = None @@ -1,3 +1,6 @@ +# # header +# coding: utf-8 + class Token(object): def __init__(self, start_mark, end_mark): @@ -1,7 +1,7 @@ [tox] # envlist = pep8,py35,py27,py34,py33,py26,pypy,jython #envlist = py35,py27,py34,py33,py26,pypy,jython -envlist = py35,py27,py34,py33,py26,pypy +envlist = pep8,py35,py27,py34,py33,py26,pypy [testenv] commands = @@ -0,0 +1,118 @@ +# coding: utf-8 + +""" +some helper functions that might be generally useful +""" + +from __future__ import print_function +from __future__ import absolute_import + +from .compat import text_type, binary_type +from .main import round_trip_load + + +# originally as comment +# https://github.com/pre-commit/pre-commit/pull/211#issuecomment-186466605 +# if you use this in your code, I suggest adding a test in your test suite +# that check this routines output against a known piece of your YAML +# before upgrades to this code break your round-tripped YAML +def load_yaml_guess_indent(stream): + # load a yaml file guess the indentation, if you use TABs ... + if isinstance(stream, text_type): + yaml_str = stream + elif isinstance(stream, binary_type): + yaml_str = stream.decode('utf-8') # most likely, but the Reader checks BOM for this + else: + yaml_str = stream.read() + indent = None # default if not found for some reason + prev_line_key_only = None + for line in yaml_str.splitlines(): + rline = line.rstrip() + if rline.startswith('- '): + idx = 1 + while line[idx] == ' ': # this will end as we rstripped + idx += 1 + if line[idx] == '#': # comment after - + continue + indent = idx + break + if rline.endswith(':'): + idx = 0 + while line[idx] == ' ': # this will end on ':' + idx += 1 + prev_line_key_only = idx + continue + if prev_line_key_only is not None and rline: + idx = 0 + while line[idx] in ' -': # this will end on ':' + idx += 1 + if idx > prev_line_key_only: + indent = idx - prev_line_key_only + break + prev_line_key_only = None + return round_trip_load(yaml_str), indent + + +def configobj_walker(cfg): + """ + walks over a ConfigObj (INI file with comments) generating + corresponding YAML output (including comments + """ + from configobj import ConfigObj + assert isinstance(cfg, ConfigObj) + for c in cfg.initial_comment: + if c.strip(): + yield c + for s in _walk_section(cfg): + if s.strip(): + yield s + for c in cfg.final_comment: + if c.strip(): + yield c + + +def _walk_section(s, level=0): + from configobj import Section + assert isinstance(s, Section) + indent = u' ' * level + for name in s.scalars: + for c in s.comments[name]: + yield indent + c.strip() + x = s[name] + if u'\n' in x: + i = indent + u' ' + x = u'|\n' + i + x.strip().replace(u'\n', u'\n' + i) + elif ':' in x: + x = u"'" + x.replace(u"'", u"''") + u"'" + line = u'{0}{1}: {2}'.format(indent, name, x) + c = s.inline_comments[name] + if c: + line += u' ' + c + yield line + for name in s.sections: + for c in s.comments[name]: + yield indent + c.strip() + line = u'{0}{1}:'.format(indent, name) + c = s.inline_comments[name] + if c: + line += u' ' + c + yield line + for val in _walk_section(s[name], level=level+1): + yield val + +# def config_obj_2_rt_yaml(cfg): +# from .comments import CommentedMap, CommentedSeq +# from configobj import ConfigObj +# assert isinstance(cfg, ConfigObj) +# #for c in cfg.initial_comment: +# # if c.strip(): +# # pass +# cm = CommentedMap() +# for name in s.sections: +# cm[name] = d = CommentedMap() +# +# +# #for c in cfg.final_comment: +# # if c.strip(): +# # yield c +# return cm |