From c9331e372f1aeef155db22fe88efeec775507c67 Mon Sep 17 00:00:00 2001 From: Anthon van der Neut Date: Thu, 18 Aug 2016 12:57:24 +0200 Subject: fix merge, iteration, reused anchors --- README.rst | 18 +++++-- __init__.py | 2 +- _test/test_anchor.py | 37 ++++++++++++++ comments.py | 137 ++++++++++++++++++++++++++++++++++++++++++++++----- composer.py | 15 ++++-- error.py | 6 ++- 6 files changed, 191 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index c54aed5..7a303d7 100644 --- a/README.rst +++ b/README.rst @@ -17,10 +17,20 @@ ChangeLog ========= :: - 0.12.3 (XXXX-XX-XX): - - correct 'in' operation for merged CommentedMaps in round-trip mode - (implementation inspired by J.Ngo, but original not working for merges) - - iteration over round-trip loaded mappings, as well as + + 0.12.3 (2016-08-17): + - correct 'in' operation for merged CommentedMaps in round-trip mode + (implementation inspired by J.Ngo, but original not working for merges) + - iteration over round-trip loaded mappings, that contain merges. Also + keys(), items(), values() (Py3/Py2) and iterkeys(), iteritems(), + itervalues(), viewkeys(), viewitems(), viewvalues() (Py2) + - reuse of anchor name now generates warning, not an error. Round-tripping such + anchors works correctly. This inherited PyYAML issue was brought to attention + by G. Coddut (and was long standing https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=515634) + suppressing the warning: + import warnings + from ruamel.yaml.error import ReusedAnchorWarning + warnings.simplefilter("ignore", ReusedAnchorWarning) 0.12.2 (2016-08-16): - minor improvements based on feedback from M. Crusoe diff --git a/__init__.py b/__init__.py index a7817d0..5e4cdfd 100644 --- a/__init__.py +++ b/__init__.py @@ -9,7 +9,7 @@ from __future__ import absolute_import _package_data = dict( full_package_name="ruamel.yaml", - version_info=(0, 12, 2, "dev"), + version_info=(0, 12, 3), author="Anthon van der Neut", author_email="a.van.der.neut@ruamel.eu", description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order", # NOQA diff --git a/_test/test_anchor.py b/_test/test_anchor.py index 1c795c0..593bf8a 100644 --- a/_test/test_anchor.py +++ b/_test/test_anchor.py @@ -9,6 +9,8 @@ from textwrap import dedent import platform from roundtrip import round_trip, dedent, round_trip_load, round_trip_dump # NOQA +from ruamel.yaml.compat import PY3 +from ruamel.yaml.error import ReusedAnchorWarning def load(s): @@ -228,6 +230,19 @@ class TestAnchorsAliases: b: 2 """) + # this is an error in PyYAML + def test_reused_anchor(self): + yaml = ''' + - &a + x: 1 + - <<: *a + - &a + x: 2 + - <<: *a + ''' + with pytest.warns(ReusedAnchorWarning): + data = round_trip(yaml) # NOQA + class TestMergeKeysValues: @@ -247,6 +262,8 @@ class TestMergeKeysValues: <<: *my """) + # in the following d always has "expanded" the merges + def test_merge_for(self): from ruamel.yaml import safe_load d = safe_load(self.yaml_str) @@ -286,3 +303,23 @@ class TestMergeKeysValues: count += 1 print(count, x) assert count == len(d[2]) + + def test_len_items_delete(self): + from ruamel.yaml import safe_load + d = safe_load(self.yaml_str) + data = round_trip_load(self.yaml_str) + x = data[2].items() + ref = len(d[2].items()) + assert len(x) == ref + del data[2]['m'] + if PY3: + ref -= 1 + assert len(x) == ref + del data[2]['d'] + if PY3: + ref -= 1 + assert len(x) == ref + del data[2]['a'] + if PY3: + ref -= 1 + assert len(x) == ref diff --git a/comments.py b/comments.py index 26587ce..c3d17b0 100644 --- a/comments.py +++ b/comments.py @@ -8,7 +8,7 @@ these are not really related, formatting could be factored out as a separate base """ -from collections import MutableSet # type: ignore +from collections import MutableSet, Sized, Set # type: ignore from ruamel.yaml.compat import ordereddict, PY2 @@ -321,6 +321,87 @@ class CommentedSeq(list, CommentedBase): return pre_comments +class CommentedMapView(Sized): + + __slots__ = '_mapping', + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + count = len(self._mapping) + done = [] # list of processed merge items, kept for masking + for merged in getattr(self._mapping, merge_attrib, []): + for x in merged[1]: + if self._mapping._unmerged_contains(x): + continue + for y in done: + if x in y: + break + else: + count += 1 + done.append(merged[1]) + return count + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class CommentedMapKeysView(CommentedMapView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + # yield from self._mapping # not in py27, pypy + for x in self._mapping: + yield x + + +class CommentedMapItemsView(CommentedMapView, Set): + + __slots__ = () + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v == value + + def __iter__(self): + for key in self._mapping._keys(): + yield (key, self._mapping[key]) + + +class CommentedMapValuesView(CommentedMapView): + + __slots__ = () + + def __contains__(self, value): + for key in self._mapping: + if value == self._mapping[key]: + return True + return False + + def __iter__(self): + print('xxy values_iter') + for key in self._mapping: + yield self._mapping[key] + + class CommentedMap(ordereddict, CommentedBase): __slots__ = [Comment.attrib, ] @@ -425,6 +506,10 @@ class CommentedMap(ordereddict, CommentedBase): return merged[1][key] raise + def _unmerged_contains(self, key): + if ordereddict.__contains__(self, key): + return True + def __contains__(self, key): if ordereddict.__contains__(self, key): return True @@ -444,6 +529,20 @@ class CommentedMap(ordereddict, CommentedBase): for x in ordereddict.__iter__(self): yield x, ordereddict.__getitem__(self, x) + def __delitem__(self, key): + found = True + for merged in getattr(self, merge_attrib, []): + try: + del merged[1][key] + found = True + except KeyError: + pass + try: + ordereddict.__delitem__(self, key) + except KeyError: + if not found: + raise + def __iter__(self): for x in ordereddict.__iter__(self): yield x @@ -477,11 +576,15 @@ class CommentedMap(ordereddict, CommentedBase): if PY2: def keys(self): return list(self._keys()) + + def iterkeys(self): + return self._keys() + + def viewkeys(self): + return CommentedMapKeysView(self) else: - # def keys(self): - # import collections - # return collections.KeysView(self._keys()) - keys = _keys + def keys(self): + return CommentedMapKeysView(self) def _values(self): for x in ordereddict.__iter__(self): @@ -501,11 +604,15 @@ class CommentedMap(ordereddict, CommentedBase): if PY2: def values(self): return list(self._values()) + + def itervalues(self): + return self._values() + + def viewvalues(self): + return CommentedMapValuesView(self) else: - # def values(self): - # import collections - # return collections.ValuesView(self) - values = _values + def values(self): + return CommentedMapValuesView(self) def _items(self): for x in ordereddict.__iter__(self): @@ -525,11 +632,15 @@ class CommentedMap(ordereddict, CommentedBase): if PY2: def items(self): return list(self._items()) + + def iteritems(self): + return self._items() + + def viewitems(self): + return CommentedMapItemsView(self) else: - # def items(self): - # import collections - # return collections.ItemsView(self._items()) - items = _items + def items(self): + return CommentedMapItemsView(self) @property def merge(self): diff --git a/composer.py b/composer.py index 320fb22..791bf50 100644 --- a/composer.py +++ b/composer.py @@ -3,8 +3,9 @@ from __future__ import absolute_import from __future__ import print_function +import warnings -from ruamel.yaml.error import MarkedYAMLError +from ruamel.yaml.error import MarkedYAMLError, ReusedAnchorWarning from ruamel.yaml.compat import utf8 from ruamel.yaml.events import ( @@ -87,10 +88,14 @@ class Composer(object): anchor = event.anchor if anchor is not None: # have an anchor if anchor in self.anchors: - raise ComposerError( - "found duplicate anchor %r; first occurence" - % utf8(anchor), self.anchors[anchor].start_mark, - "second occurence", event.start_mark) + # raise ComposerError( + # "found duplicate anchor %r; first occurence" + # % utf8(anchor), self.anchors[anchor].start_mark, + # "second occurence", event.start_mark) + ws = "\nfound duplicate anchor {!r}\nfirst occurence {}\nsecond occurence "\ + "{}".format( + (anchor), self.anchors[anchor].start_mark, event.start_mark) + warnings.warn(ws, ReusedAnchorWarning) self.descend_resolver(parent, index) if self.check_event(ScalarEvent): node = self.compose_scalar_node(anchor) diff --git a/error.py b/error.py index fb5e78c..e93c20e 100644 --- a/error.py +++ b/error.py @@ -4,7 +4,7 @@ from __future__ import absolute_import from ruamel.yaml.compat import utf8 -__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError', 'ReusedAnchorWarning'] class Mark(object): @@ -80,3 +80,7 @@ class MarkedYAMLError(YAMLError): if self.note is not None: lines.append(self.note) return '\n'.join(lines) + + +class ReusedAnchorWarning(Warning): + pass -- cgit v1.2.1