diff options
| -rw-r--r-- | CHANGES | 5 | ||||
| -rw-r--r-- | README.rst | 5 | ||||
| -rw-r--r-- | __init__.py | 12 | ||||
| -rw-r--r-- | _test/data/duplicate-key.former-loader-error.code | 1 | ||||
| -rw-r--r-- | _test/data/duplicate-key.former-loader-error.data | 3 | ||||
| -rw-r--r-- | _test/data/duplicate-mapping-key.former-loader-error.code | 1 | ||||
| -rw-r--r-- | _test/data/duplicate-mapping-key.former-loader-error.data | 6 | ||||
| -rw-r--r-- | _test/data/duplicate-value-key.former-loader-error.code | 1 | ||||
| -rw-r--r-- | _test/data/duplicate-value-key.former-loader-error.data | 4 | ||||
| -rw-r--r-- | _test/test_anchor.py | 26 | ||||
| -rw-r--r-- | _test/test_api_change.py | 28 | ||||
| -rw-r--r-- | compat.py | 10 | ||||
| -rw-r--r-- | constructor.py | 100 | ||||
| -rw-r--r-- | error.py | 94 | ||||
| -rw-r--r-- | main.py | 7 | ||||
| -rw-r--r-- | nodes.py | 2 | ||||
| -rw-r--r-- | tox.ini | 2 |
17 files changed, 247 insertions, 60 deletions
@@ -1,3 +1,8 @@ +[0, 15, 1]: 2017-06-07 + - `duplicate keys <http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys>`_ + in mappings generate an error (in the old API this change generates a warning until 0.16) + - dependecy on ruamel.ordereddict for 2.7 now via extras_require + [0, 15, 0]: 2017-06-04 - it is no allowed to pass in a ``pathlib.Path`` as "stream" parameter to all load/dump functions @@ -32,6 +32,11 @@ ChangeLog .. should insert NEXT: at the beginning of line for next key +0.15.1 (2017-06-07): + - `duplicate keys <http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys>`_ + in mappings generate an error (in the old API this change generates a warning until 0.16) + - dependecy on ruamel.ordereddict for 2.7 now via extras_require + 0.15.0 (2017-06-04): - it is no allowed to pass in a ``pathlib.Path`` as "stream" parameter to all load/dump functions diff --git a/__init__.py b/__init__.py index 64f9a9f..bcec717 100644 --- a/__init__.py +++ b/__init__.py @@ -11,16 +11,15 @@ if False: # MYPY _package_data = dict( full_package_name='ruamel.yaml', - version_info=(0, 15, 1, 'dev'), - __version__='0.15.1.dev', + version_info=(0, 15, 1), + __version__='0.15.1', author='Anthon van der Neut', author_email='a.van.der.neut@ruamel.eu', description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order', # NOQA entry_points=None, - install_requires=dict( - any=[], - py27=['ruamel.ordereddict'], - ), + extras_require={':platform_python_implementation=="CPython" and python_version<="2.7"': [ + 'ruamel.ordereddict', + ]}, ext_modules=[dict( name='_ruamel_yaml', src=['ext/_ruamel_yaml.c', 'ext/api.c', 'ext/writer.c', 'ext/dumper.c', @@ -45,6 +44,7 @@ _package_data = dict( 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Text Processing :: Markup', ], + keywords='yaml 1.2 parser round-trip preserve quotes order config', windows_wheels=True, read_the_docs='yaml', many_linux='libyaml-devel', diff --git a/_test/data/duplicate-key.former-loader-error.code b/_test/data/duplicate-key.former-loader-error.code deleted file mode 100644 index cb73906..0000000 --- a/_test/data/duplicate-key.former-loader-error.code +++ /dev/null @@ -1 +0,0 @@ -{ 'foo': 'baz' } diff --git a/_test/data/duplicate-key.former-loader-error.data b/_test/data/duplicate-key.former-loader-error.data deleted file mode 100644 index 84deb8f..0000000 --- a/_test/data/duplicate-key.former-loader-error.data +++ /dev/null @@ -1,3 +0,0 @@ ---- -foo: bar -foo: baz diff --git a/_test/data/duplicate-mapping-key.former-loader-error.code b/_test/data/duplicate-mapping-key.former-loader-error.code deleted file mode 100644 index 17a6285..0000000 --- a/_test/data/duplicate-mapping-key.former-loader-error.code +++ /dev/null @@ -1 +0,0 @@ -{ 'foo': { 'baz': 'bat', 'foo': 'duplicate key' } } diff --git a/_test/data/duplicate-mapping-key.former-loader-error.data b/_test/data/duplicate-mapping-key.former-loader-error.data deleted file mode 100644 index 7e7b4d1..0000000 --- a/_test/data/duplicate-mapping-key.former-loader-error.data +++ /dev/null @@ -1,6 +0,0 @@ ---- -&anchor foo: - foo: bar - *anchor: duplicate key - baz: bat - *anchor: duplicate key diff --git a/_test/data/duplicate-value-key.former-loader-error.code b/_test/data/duplicate-value-key.former-loader-error.code deleted file mode 100644 index 12f48c1..0000000 --- a/_test/data/duplicate-value-key.former-loader-error.code +++ /dev/null @@ -1 +0,0 @@ -{ 'foo': 'bar', '=': 2 } diff --git a/_test/data/duplicate-value-key.former-loader-error.data b/_test/data/duplicate-value-key.former-loader-error.data deleted file mode 100644 index b34a1d6..0000000 --- a/_test/data/duplicate-value-key.former-loader-error.data +++ /dev/null @@ -1,4 +0,0 @@ ---- -=: 1 -foo: bar -=: 2 diff --git a/_test/test_anchor.py b/_test/test_anchor.py index 05c8693..fc4a881 100644 --- a/_test/test_anchor.py +++ b/_test/test_anchor.py @@ -11,6 +11,7 @@ import platform from roundtrip import round_trip, dedent, round_trip_load, round_trip_dump # NOQA from ruamel.yaml.compat import PY3 from ruamel.yaml.error import ReusedAnchorWarning +from ruamel.yaml import version_info def load(s): @@ -325,6 +326,31 @@ class TestMergeKeysValues: assert len(x) == ref +class TestDuplicateKeyThroughAnchor: + def test_duplicate_key_00(self): + from ruamel.yaml import safe_load, round_trip_load + from ruamel.yaml.constructor import DuplicateKeyFutureWarning, DuplicateKeyError + s = dedent("""\ + &anchor foo: + foo: bar + *anchor : duplicate key + baz: bat + *anchor : duplicate key + """) + if version_info < (0, 15, 1): + pass + elif version_info < (0, 16, 0): + with pytest.warns(DuplicateKeyFutureWarning): + safe_load(s) + with pytest.warns(DuplicateKeyFutureWarning): + round_trip_load(s) + else: + with pytest.raises(DuplicateKeyError): + safe_load(s) + with pytest.raises(DuplicateKeyError): + round_trip_load(s) + + class TestFullCharSetAnchors: def test_master_of_orion(self): # https://bitbucket.org/ruamel/yaml/issues/72/not-allowed-in-anchor-names diff --git a/_test/test_api_change.py b/_test/test_api_change.py new file mode 100644 index 0000000..10997f4 --- /dev/null +++ b/_test/test_api_change.py @@ -0,0 +1,28 @@ +# coding: utf-8 + +""" +testing of anchors and the aliases referring to them +""" + +import pytest +from ruamel.yaml import YAML +from ruamel.yaml.constructor import DuplicateKeyError + + +class TestNewAPI: + def test_duplicate_keys_00(self): + from ruamel.yaml.constructor import DuplicateKeyError + yaml = YAML() + with pytest.raises(DuplicateKeyError): + yaml.load('{a: 1, a: 2}') + + def test_duplicate_keys_01(self): + yaml = YAML(typ='safe', pure=True) + with pytest.raises(DuplicateKeyError): + yaml.load('{a: 1, a: 2}') + + # @pytest.mark.xfail(strict=True) + def test_duplicate_keys_02(self): + yaml = YAML(typ='safe') + with pytest.raises(DuplicateKeyError): + yaml.load('{a: 1, a: 2}') @@ -180,3 +180,13 @@ def check_anchorname_char(ch): if ch in u',[]{}': return False return check_namespace_char(ch) + + +def version_tnf(t1, t2=None): + # type: (Any, Any) -> Any + from ruamel.yaml import version_info # NOQA + if version_info < t1: + return True + if t2 is not None and version_info < t2: + return None + return False diff --git a/constructor.py b/constructor.py index 4a52b27..37a7a8f 100644 --- a/constructor.py +++ b/constructor.py @@ -9,12 +9,13 @@ import binascii import re import sys import types +import warnings -from ruamel.yaml.error import (MarkedYAMLError) +from ruamel.yaml.error import (MarkedYAMLError, MarkedYAMLFutureWarning) from ruamel.yaml.nodes import * # NOQA from ruamel.yaml.nodes import (SequenceNode, MappingNode, ScalarNode) from ruamel.yaml.compat import (utf8, builtins_module, to_str, PY2, PY3, # NOQA - ordereddict, text_type, nprint) + ordereddict, text_type, nprint, version_tnf) from ruamel.yaml.comments import * # NOQA from ruamel.yaml.comments import (CommentedMap, CommentedOrderedMap, CommentedSet, CommentedKeySeq, CommentedSeq) @@ -36,6 +37,14 @@ class ConstructorError(MarkedYAMLError): pass +class DuplicateKeyFutureWarning(MarkedYAMLFutureWarning): + pass + + +class DuplicateKeyError(MarkedYAMLFutureWarning): + pass + + class BaseConstructor(object): yaml_constructors = {} # type: Dict[Any, Any] @@ -52,6 +61,7 @@ class BaseConstructor(object): self.state_generators = [] # type: List[Any] self.deep_construct = False self._preserve_quotes = preserve_quotes + self.allow_duplicate_keys = version_tnf((0, 15, 1), (0, 16)) @property def composer(self): @@ -193,31 +203,62 @@ class BaseConstructor(object): None, None, "expected a mapping node, but found %s" % node.id, node.start_mark) - mapping = {} - for key_node, value_node in node.value: - # keys can be list -> deep - key = self.construct_object(key_node, deep=True) - # lists are not hashable, but tuples are - if not isinstance(key, collections.Hashable): # type: ignore - if isinstance(key, list): - key = tuple(key) - if PY2: - try: - hash(key) - except TypeError as exc: - raise ConstructorError( - "while constructing a mapping", node.start_mark, - "found unacceptable key (%s)" % - exc, key_node.start_mark) - else: - if not isinstance(key, collections.Hashable): - raise ConstructorError( - "while constructing a mapping", node.start_mark, - "found unhashable key", key_node.start_mark) - - value = self.construct_object(value_node, deep=deep) - mapping[key] = value - return mapping + total_mapping = {} + if hasattr(node, 'merge'): + todo = [(node.merge, False), (node.value, True)] + else: + todo = [(node.value, True)] + for values, check in todo: + mapping = {} + for key_node, value_node in values: + # keys can be list -> deep + key = self.construct_object(key_node, deep=True) + # lists are not hashable, but tuples are + if not isinstance(key, collections.Hashable): # type: ignore + if isinstance(key, list): + key = tuple(key) + if PY2: + try: + hash(key) + except TypeError as exc: + raise ConstructorError( + "while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % + exc, key_node.start_mark) + else: + if not isinstance(key, collections.Hashable): + raise ConstructorError( + "while constructing a mapping", node.start_mark, + "found unhashable key", key_node.start_mark) + + value = self.construct_object(value_node, deep=deep) + if check: + self.check_mapping_key(node, key_node, mapping, key, value) + mapping[key] = value + total_mapping.update(mapping) + return total_mapping + + def check_mapping_key(self, node, key_node, mapping, key, value): + if key in mapping: + if not self.allow_duplicate_keys: + args = [ + "while constructing a mapping", node.start_mark, + 'found duplicate key "{}" with value "{}" ' + '(original value: "{}")'.format( + key, value, mapping[key]), key_node.start_mark, + """ + To suppress this check see: + http://yaml.readthedocs.io/en/latest/api.html#duplicate-keys + """, + """\ + Duplicate keys will become and error in future releases, and are errors + by default when using the new API. + """, + ] + if self.allow_duplicate_keys is None: + warnings.warn(DuplicateKeyFutureWarning(*args)) + else: + raise DuplicateKeyError(*args) def construct_pairs(self, node, deep=False): # type: (Any, bool) -> Any @@ -299,7 +340,8 @@ class SafeConstructor(BaseConstructor): else: index += 1 if bool(merge): - node.value = merge + node.value + node.merge = merge # separate merge keys to be able to update without duplicate + # node.value = merge + node.value def construct_mapping(self, node, deep=False): # type: (Any, bool) -> Any @@ -1140,6 +1182,8 @@ class RoundTripConstructor(SafeConstructor): "while constructing a mapping", node.start_mark, "found unhashable key", key_node.start_mark) value = self.construct_object(value_node, deep=deep) + self.check_mapping_key(node, key_node, maptyp, key, value) + if key_node.comment and len(key_node.comment) > 4 and \ key_node.comment[4]: if last_value is None: @@ -3,6 +3,7 @@ from __future__ import absolute_import import warnings +import textwrap from ruamel.yaml.compat import utf8 @@ -12,7 +13,8 @@ if False: # MYPY __all__ = [ 'FileMark', 'StringMark', 'CommentMark', 'YAMLError', 'MarkedYAMLError', - 'ReusedAnchorWarning', 'UnsafeLoaderWarning', + 'ReusedAnchorWarning', 'UnsafeLoaderWarning', 'MarkedYAMLWarning', + 'MarkedYAMLFutureWarning', ] @@ -98,13 +100,14 @@ class YAMLError(Exception): class MarkedYAMLError(YAMLError): def __init__(self, context=None, context_mark=None, - problem=None, problem_mark=None, note=None): + problem=None, problem_mark=None, note=None, warn=None): # type: (Any, Any, Any, Any, Any) -> None self.context = context self.context_mark = context_mark self.problem = problem self.problem_mark = problem_mark self.note = note + # warn is ignored def __str__(self): # type: () -> Any @@ -121,8 +124,9 @@ class MarkedYAMLError(YAMLError): lines.append(self.problem) if self.problem_mark is not None: lines.append(str(self.problem_mark)) - if self.note is not None: - lines.append(self.note) + if self.note is not None and self.note: + note = textwrap.dedent(self.note) + lines.append(note) return '\n'.join(lines) @@ -130,11 +134,49 @@ class YAMLStreamError(Exception): pass -class ReusedAnchorWarning(Warning): +class YAMLWarning(Warning): pass -class UnsafeLoaderWarning(Warning): +class MarkedYAMLWarning(YAMLWarning): + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None, warn=None): + # type: (Any, Any, Any, Any, Any) -> None + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.warn = warn + + def __str__(self): + # type: () -> Any + lines = [] # type: List[str] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None or + self.context_mark.name != self.problem_mark.name or + self.context_mark.line != self.problem_mark.line or + self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None and self.note: + note = textwrap.dedent(self.note) + lines.append(note) + if self.warn is not None and self.warn: + warn = textwrap.dedent(self.warn) + lines.append(warn) + return '\n'.join(lines) + + +class ReusedAnchorWarning(YAMLWarning): + pass + + +class UnsafeLoaderWarning(YAMLWarning): text = """ The default 'Loader' for 'load(stream)' without further arguments can be unsafe. Use 'load(stream, Loader=ruamel.yaml.Loader)' explicitly if that is OK. @@ -147,3 +189,43 @@ In most other cases you should consider using 'safe_load(stream)'""" pass warnings.simplefilter('once', UnsafeLoaderWarning) + + +class YAMLFutureWarning(Warning): + pass + + +class MarkedYAMLFutureWarning(YAMLFutureWarning): + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None, warn=None): + # type: (Any, Any, Any, Any, Any) -> None + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + self.warn = warn + + def __str__(self): + # type: () -> Any + lines = [] # type: List[str] + if self.context is not None: + lines.append(self.context) + + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None or + self.context_mark.name != self.problem_mark.name or + self.context_mark.line != self.problem_mark.line or + self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None and self.note: + note = textwrap.dedent(self.note) + lines.append(note) + if self.warn is not None and self.warn: + warn = textwrap.dedent(self.warn) + lines.append(warn) + return '\n'.join(lines) @@ -99,6 +99,7 @@ class YAML(object): self.prefix_colon = None self.version = None self.preserve_quotes = None + self.allow_duplicate_keys = False # duplicate keys in map, set self.encoding = None self.explicit_start = None self.explicit_end = None @@ -157,8 +158,9 @@ class YAML(object): # type: () -> Any attr = '_' + sys._getframe().f_code.co_name if not hasattr(self, attr): - setattr(self, attr, self.Constructor( - preserve_quotes=self.preserve_quotes, loader=self)) + cnst = self.Constructor(preserve_quotes=self.preserve_quotes, loader=self) + cnst.allow_duplicate_keys = self.allow_duplicate_keys + setattr(self, attr, cnst) return getattr(self, attr) @property @@ -289,6 +291,7 @@ class YAML(object): CParser.__init__(selfx, stream) selfx._parser = selfx._composer = selfx self.Constructor.__init__(selfx, loader=selfx) + selfx.allow_duplicate_keys = self.allow_duplicate_keys rslvr.__init__(selfx, loadumper=selfx) self._stream = stream loader = XLoader(stream) @@ -97,5 +97,5 @@ class SequenceNode(CollectionNode): class MappingNode(CollectionNode): - __slots__ = () + __slots__ = ('merge', ) id = 'mapping' @@ -1,7 +1,7 @@ [tox] # envlist = pep8,py35,py27,py34,py33,py26,pypy,jython #envlist = py35,py27,py34,py33,py26,pypy,jython -envlist = pep8,py35,py36,py27,py34,pypy +envlist = pep8,py36,py27,py35,py34,pypy [testenv] commands = |