diff options
| -rw-r--r-- | README.rst | 52 | ||||
| -rw-r--r-- | __init__.py | 2 | ||||
| -rw-r--r-- | _test/test_version.py | 105 | ||||
| -rw-r--r-- | constructor.py | 16 | ||||
| -rw-r--r-- | cyaml.py | 6 | ||||
| -rw-r--r-- | emitter.py | 13 | ||||
| -rw-r--r-- | loader.py | 18 | ||||
| -rw-r--r-- | main.py | 33 | ||||
| -rw-r--r-- | resolver.py | 154 |
9 files changed, 368 insertions, 31 deletions
@@ -2,6 +2,13 @@ ruamel.yaml =========== + +**Starting with 0.11.0 the RoundTripLoader differentiates +between YAML 1.2 and YAML 1.1. This may cause compatibility problems, +see the "Document version support" section below.** + +---- + Starting with 0.10.7 the package has been reorganised and the command line utility is in its own package ``ruamel.yaml.cmd`` (so installing ``ruamel.yaml`` doesn't pull in possibly irrelevant modules @@ -45,6 +52,42 @@ Major differences with PyYAML 3.11: (``lc.key('a')``, ``lc.value('a')`` resp. ``lc.item(3)``) - preservation of whitelines after block scalars. Contributed by Sam Thursfield. +Document version support. +========================= + +In YAML a document version can be explicitly set by using:: + + %YAML 1.x + +before the document start (at the top or before a +``---``). For ``ruamel.yaml`` x has to be 1 or 2. If no explicit +version is set `version 1.2 <http://www.yaml.org/spec/1.2/spec.html>`_ +is assumed (which has been released in 2009). + +The 1.2 version does **not** support: + +- sexagesimals like ``12:34:56`` +- octals that start with 0 only: like ``012`` for number 10 (``0o12`` **is** + supported by YAML 1.2) +- Unquoted Yes and On as alternatives for True and No and Off for False. + +If you cannot change your YAML files and you need them to load as 1.1 +you can load with: + + ruamel.yaml.load(some_str, Loader=ruamel.yaml.RoundTripLoader, version=(1, 1)) + +or the equivalent (version can be a tuple, list or string): + + ruamel.yaml.round_trip_load(some_str, version="1.1") + +this also works for ``load_all``/``round_trip_load_all``. + +*If you cannot change your code, stick with ruamel.yaml==0.10.23 and let +me know if it would help to be able to set an environment variable.* + +This does not affect dump as ruamel.yaml never emitted sexagesimals, nor +octal numbers, and emitted booleans always as true resp. false + Round trip including comments ============================= @@ -272,3 +315,12 @@ Testing Testing is done using `tox <https://pypi.python.org/pypi/tox>`_, which uses `virtualenv <https://pypi.python.org/pypi/virtualenv>`_ and `pytest <http://pytest.org/latest/>`_. + +ChangeLog +========= + +:: + + 0.11.0 (2016-02-18): + - RoundTripLoader loads 1.2 by default (no sexagesimals, 012 octals nor + yes/no/on/off booleans diff --git a/__init__.py b/__init__.py index f9193ef..7ff4a96 100644 --- a/__init__.py +++ b/__init__.py @@ -9,7 +9,7 @@ from __future__ import absolute_import _package_data = dict( full_package_name="ruamel.yaml", - version_info=(0, 10, 23), + version_info=(0, 11, 0), author="Anthon van der Neut", author_email="a.van.der.neut@ruamel.eu", description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order", # NOQA diff --git a/_test/test_version.py b/_test/test_version.py new file mode 100644 index 0000000..55d80c3 --- /dev/null +++ b/_test/test_version.py @@ -0,0 +1,105 @@ +# coding: utf-8 + +import pytest # NOQA + +import ruamel.yaml +from roundtrip import dedent + +def load(s, version=None): + return ruamel.yaml.round_trip_load(dedent(s), version) + + + +class TestVersions: + def test_explicit_1_2(self): + l = load("""\ + %YAML 1.2 + --- + - 12:34:56 + - 012 + - 012345678 + - 0o12 + - on + - off + - yes + - no + - true + """) + assert l[0] == '12:34:56' + assert l[1] == 12 + assert l[2] == '012345678' + assert l[3] == 10 + assert l[4] == 'on' + assert l[5] == 'off' + assert l[6] == 'yes' + assert l[7] == 'no' + assert l[8] is True + + def test_explicit_1_1(self): + l = load("""\ + %YAML 1.1 + --- + - 12:34:56 + - 012 + - 012345678 + - 0o12 + - on + - off + - yes + - no + - true + """) + assert l[0] == 45296 + assert l[1] == 10 + assert l[2] == '012345678' + assert l[3] == 10 + assert l[4] is True + assert l[5] is False + assert l[6] is True + assert l[7] is False + assert l[8] is True + + def test_implicit_1_2(self): + l = load("""\ + - 12:34:56 + - 012 + - 012345678 + - 0o12 + - on + - off + - yes + - no + - true + """) + assert l[0] == '12:34:56' + assert l[1] == 12 + assert l[2] == '012345678' + assert l[3] == 10 + assert l[4] == 'on' + assert l[5] == 'off' + assert l[6] == 'yes' + assert l[7] == 'no' + assert l[8] is True + + def test_load_version_1_1(self): + l = load("""\ + - 12:34:56 + - 012 + - 012345678 + - 0o12 + - on + - off + - yes + - no + - true + """, version="1.1") + assert l[0] == 45296 + assert l[1] == 10 + assert l[2] == '012345678' + assert l[3] == 10 + assert l[4] is True + assert l[5] is False + assert l[6] is True + assert l[7] is False + assert l[8] is True + diff --git a/constructor.py b/constructor.py index c3b5a59..9ae2e23 100644 --- a/constructor.py +++ b/constructor.py @@ -73,6 +73,9 @@ class BaseConstructor(object): return data def construct_object(self, node, deep=False): + """deep is True when creating an object/mapping recursively, + in that case want the underlying elements available during construction + """ if node in self.constructed_objects: return self.constructed_objects[node] if deep: @@ -132,6 +135,9 @@ class BaseConstructor(object): return node.value def construct_sequence(self, node, deep=False): + """deep is True when creating an object/mapping recursively, + in that case want the underlying elements available during construction + """ if not isinstance(node, SequenceNode): raise ConstructorError( None, None, @@ -141,6 +147,9 @@ class BaseConstructor(object): for child in node.value] def construct_mapping(self, node, deep=False): + """deep is True when creating an object/mapping recursively, + in that case want the underlying elements available during construction + """ if not isinstance(node, MappingNode): raise ConstructorError( None, None, @@ -250,6 +259,9 @@ class SafeConstructor(BaseConstructor): node.value = merge + node.value def construct_mapping(self, node, deep=False): + """deep is True when creating an object/mapping recursively, + in that case want the underlying elements available during construction + """ if isinstance(node, MappingNode): self.flatten_mapping(node) return BaseConstructor.construct_mapping(self, node, deep=deep) @@ -288,9 +300,9 @@ class SafeConstructor(BaseConstructor): return sign*int(value[2:], 16) elif value.startswith('0o'): return sign*int(value[2:], 8) - elif value[0] == '0': + elif self.processing_version != (1, 2) and value[0] == '0': return sign*int(value, 8) - elif ':' in value: + elif self.processing_version != (1, 2) and ':' in value: digits = [int(part) for part in value.split(':')] digits.reverse() base = 1 @@ -18,21 +18,21 @@ except (ImportError, ValueError): # for Jython class CBaseLoader(CParser, BaseConstructor, BaseResolver): - def __init__(self, stream): + def __init__(self, stream, version=None): CParser.__init__(self, stream) BaseConstructor.__init__(self) BaseResolver.__init__(self) class CSafeLoader(CParser, SafeConstructor, Resolver): - def __init__(self, stream): + def __init__(self, stream, version=None): CParser.__init__(self, stream) SafeConstructor.__init__(self) Resolver.__init__(self) class CLoader(CParser, Constructor, Resolver): - def __init__(self, stream): + def __init__(self, stream, version=None): CParser.__init__(self, stream) Constructor.__init__(self) Resolver.__init__(self) @@ -1218,17 +1218,18 @@ class Emitter(object): col = self.column + 1 # print('post_comment', self.line, self.column, value) try: - if PY2: - value = value.encode('utf-8') - except UnicodeDecodeError: - pass - try: # at least one space if the current column >= the start column of the comment # but not at the start of a line nr_spaces = col - self.column if self.column and value.strip() and nr_spaces < 1: nr_spaces = 1 - self.stream.write(' ' * nr_spaces + value) + value = ' ' * nr_spaces + value + try: + if PY2: + value = value.encode('utf-8') + except UnicodeDecodeError: + pass + self.stream.write(value) except TypeError: print('TypeError while trying to write', repr(value), type(value)) raise @@ -18,9 +18,8 @@ except (ImportError, ValueError): # for Jython from ruamel.yaml.resolver import * # NOQA -class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, - BaseResolver): - def __init__(self, stream): +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + def __init__(self, stream, version=None): Reader.__init__(self, stream) Scanner.__init__(self) Parser.__init__(self) @@ -30,7 +29,7 @@ class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): - def __init__(self, stream): + def __init__(self, stream, version=None): Reader.__init__(self, stream) Scanner.__init__(self) Parser.__init__(self) @@ -40,8 +39,7 @@ class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): - - def __init__(self, stream): + def __init__(self, stream, version=None): Reader.__init__(self, stream) Scanner.__init__(self) Parser.__init__(self) @@ -50,12 +48,12 @@ class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): Resolver.__init__(self) -class RoundTripLoader(Reader, RoundTripScanner, Parser, - Composer, RoundTripConstructor, Resolver): - def __init__(self, stream): +class RoundTripLoader(Reader, RoundTripScanner, Parser, Composer, + RoundTripConstructor, VersionedResolver): + def __init__(self, stream, version=None): Reader.__init__(self, stream) RoundTripScanner.__init__(self) Parser.__init__(self) Composer.__init__(self) RoundTripConstructor.__init__(self) - Resolver.__init__(self) + VersionedResolver.__init__(self, version) @@ -65,24 +65,24 @@ def compose_all(stream, Loader=Loader): loader.dispose() -def load(stream, Loader=Loader): +def load(stream, Loader=Loader, version=None): """ Parse the first YAML document in a stream and produce the corresponding Python object. """ - loader = Loader(stream) + loader = Loader(stream, version) try: return loader.get_single_data() finally: loader.dispose() -def load_all(stream, Loader=Loader): +def load_all(stream, Loader=Loader, version=None): """ Parse all YAML documents in a stream and produce corresponding Python objects. """ - loader = Loader(stream) + loader = Loader(stream, version) try: while loader.check_data(): yield loader.get_data() @@ -90,22 +90,39 @@ def load_all(stream, Loader=Loader): loader.dispose() -def safe_load(stream): +def safe_load(stream, version=None): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + Resolve only basic YAML tags. + """ + return load(stream, SafeLoader, version) + + +def safe_load_all(stream, version=None): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + Resolve only basic YAML tags. + """ + return load_all(stream, SafeLoader, version) + +def round_trip_load(stream, version=None): """ Parse the first YAML document in a stream and produce the corresponding Python object. Resolve only basic YAML tags. """ - return load(stream, SafeLoader) + return load(stream, RoundTripLoader, version) -def safe_load_all(stream): +def round_trip_load_all(stream, version=None): """ Parse all YAML documents in a stream and produce corresponding Python objects. Resolve only basic YAML tags. """ - return load_all(stream, SafeLoader) + return load_all(stream, RoundTripLoader, version) def emit(events, stream=None, Dumper=Dumper, diff --git a/resolver.py b/resolver.py index 04dfcf4..44b7b32 100644 --- a/resolver.py +++ b/resolver.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -__all__ = ['BaseResolver', 'Resolver'] +__all__ = ['BaseResolver', 'Resolver', 'VersionedResolver'] try: from .error import * # NOQA @@ -15,6 +15,8 @@ except (ImportError, ValueError): # for Jython import re +_DEFAULT_VERSION = (1, 2) + class ResolverError(YAMLError): pass @@ -29,6 +31,7 @@ class BaseResolver(object): yaml_path_resolvers = {} def __init__(self): + self._loader_version = None self.resolver_exact_paths = [] self.resolver_prefix_paths = [] @@ -174,6 +177,9 @@ class BaseResolver(object): elif kind is MappingNode: return self.DEFAULT_MAPPING_TAG + @property + def processing_version(self): + return None class Resolver(BaseResolver): pass @@ -237,3 +243,149 @@ Resolver.add_implicit_resolver( u'tag:yaml.org,2002:yaml', re.compile(u'^(?:!|&|\\*)$'), list(u'!&*')) + +# resolvers consist of +# - a list of applicable version +# - a tag +# - a regexp +# - a list of first characters to match +implicit_resolvers = [ + ([(1, 2)], + u'tag:yaml.org,2002:bool', + re.compile(u'''^(?:true|True|TRUE|false|False|FALSE)$''', re.X), + list(u'tTfF')), + ([(1, 1)], + u'tag:yaml.org,2002:bool', + re.compile(u'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list(u'yYnNtTfFoO')), + ([(1, 2), (1, 1)], + u'tag:yaml.org,2002:float', + re.compile(u'''^(?: + [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? + |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) + |\\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]* + |[-+]?\\.(?:inf|Inf|INF) + |\\.(?:nan|NaN|NAN))$''', re.X), + list(u'-+0123456789.')), + ([(1, 2)], + u'tag:yaml.org,2002:int', + re.compile(u'''^(?:[-+]?0b[0-1_]+ + |[-+]?0o?[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+)$''', re.X), + list(u'-+0123456789')), + ([(1, 1)], + u'tag:yaml.org,2002:int', + re.compile(u'''^(?:[-+]?0b[0-1_]+ + |[-+]?0o?[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list(u'-+0123456789')), + ([(1, 2), (1, 1)], + u'tag:yaml.org,2002:merge', + re.compile(u'^(?:<<)$'), + [u'<']), + ([(1, 2), (1, 1)], + u'tag:yaml.org,2002:null', + re.compile(u'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + [u'~', u'n', u'N', u'']), + ([(1, 2), (1, 1)], + u'tag:yaml.org,2002:timestamp', + re.compile(u'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \\t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\\.[0-9]*)? + (?:[ \\t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list(u'0123456789')), + ([(1, 2), (1, 1)], + u'tag:yaml.org,2002:value', + re.compile(u'^(?:=)$'), + [u'=']), + # The following resolver is only for documentation purposes. It cannot work + # because plain scalars cannot start with '!', '&', or '*'. + ([(1, 2), (1, 1)], + u'tag:yaml.org,2002:yaml', + re.compile(u'^(?:!|&|\\*)$'), + list(u'!&*')), +] + + +class VersionedResolver(BaseResolver): + """ + contrary to the "normal" resolver, the smart resolver delays loading + the pattern matching rules. That way it can decide to load 1.1 rules + or the (default) 1.2 that no longer support octal without 0o, sexagesimals + and Yes/No/On/Off booleans. + """ + + def __init__(self, version=None): + BaseResolver.__init__(self) + self._loader_version = self.get_loader_version(version) + self._version_implicit_resolver = {} + + def add_version_implicit_resolver(self, version, tag, regexp, first): + if first is None: + first = [None] + impl_resolver = self._version_implicit_resolver.setdefault(version, {}) + for ch in first: + impl_resolver.setdefault(ch, []).append((tag, regexp)) + + def get_loader_version(self, version): + if version is None or isinstance(version, tuple): + return version + if isinstance(version, list): + return tuple(version) + # assume string + return tuple(map(int, version.split(u'.'))) + + @property + def resolver(self): + """ + select the resolver based on the version we are parsing + """ + version = self.processing_version + if version not in self._version_implicit_resolver: + print('>>> version', self.yaml_version, version) + for x in implicit_resolvers: + if version in x[0]: + self.add_version_implicit_resolver(version, x[1], x[2], x[3]) + return self._version_implicit_resolver[version] + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == u'': + resolvers = self.resolver.get(u'', []) + else: + resolvers = self.resolver.get(value[0], []) + resolvers += self.resolver.get(None, []) + for tag, regexp in resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + if self.yaml_path_resolvers: + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + + @property + def processing_version(self): + version = self.yaml_version + if version is None: + version = self._loader_version + if version is None: + version = _DEFAULT_VERSION + return version |