diff options
| -rw-r--r-- | README.rst | 7 | ||||
| -rw-r--r-- | _test/test_anchor.py | 61 | ||||
| -rw-r--r-- | comments.py | 96 | ||||
| -rw-r--r-- | constructor.py | 6 | ||||
| -rw-r--r-- | representer.py | 8 |
5 files changed, 168 insertions, 10 deletions
@@ -17,9 +17,10 @@ ChangeLog ========= :: - - 0.12.3 (2016-08-XX): - - proper 'in' operation for merged CommentedMaps (implementation by J.Ngo) + 0.12.3 (XXXX-XX-XX): + - correct 'in' operation for merged CommentedMaps in round-trip mode + (implementation inspired by J.Ngo, but original not working for merges) + - iteration over round-trip loaded mappings, as well as 0.12.2 (2016-08-16): - minor improvements based on feedback from M. Crusoe diff --git a/_test/test_anchor.py b/_test/test_anchor.py index 929b1ed..1c795c0 100644 --- a/_test/test_anchor.py +++ b/_test/test_anchor.py @@ -175,7 +175,7 @@ class TestAnchorsAliases: k: &level_2 { a: 1, b2 } l: &level_1 { a: 10, c: 3 } m: - << : *level_1 + <<: *level_1 c: 30 d: 40 """) @@ -227,3 +227,62 @@ class TestAnchorsAliases: c: 3 b: 2 """) + + +class TestMergeKeysValues: + + yaml_str = dedent("""\ + - &mx + a: x1 + b: x2 + c: x3 + - &my + a: y1 + b: y2 # masked by the one in &mx + d: y4 + - + a: 1 + <<: *mx + m: 6 + <<: *my + """) + + def test_merge_for(self): + from ruamel.yaml import safe_load + d = safe_load(self.yaml_str) + data = round_trip_load(self.yaml_str) + count = 0 + for x in data[2]: + count += 1 + print(count, x) + assert count == len(d[2]) + + def test_merge_keys(self): + from ruamel.yaml import safe_load + d = safe_load(self.yaml_str) + data = round_trip_load(self.yaml_str) + count = 0 + for x in data[2].keys(): + count += 1 + print(count, x) + assert count == len(d[2]) + + def test_merge_values(self): + from ruamel.yaml import safe_load + d = safe_load(self.yaml_str) + data = round_trip_load(self.yaml_str) + count = 0 + for x in data[2].values(): + count += 1 + print(count, x) + assert count == len(d[2]) + + def test_merge_items(self): + from ruamel.yaml import safe_load + d = safe_load(self.yaml_str) + data = round_trip_load(self.yaml_str) + count = 0 + for x in data[2].items(): + count += 1 + print(count, x) + assert count == len(d[2]) diff --git a/comments.py b/comments.py index 07e2e9b..26587ce 100644 --- a/comments.py +++ b/comments.py @@ -10,7 +10,7 @@ a separate base from collections import MutableSet # type: ignore -from ruamel.yaml.compat import ordereddict +from ruamel.yaml.compat import ordereddict, PY2 __all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap", "CommentedSet", 'comment_attrib', 'merge_attrib'] @@ -24,7 +24,7 @@ tag_attrib = '_yaml_tag' class Comment(object): - # sys.getsize tested the Comment objects, __slots__ make them bigger + # sys.getsize tested the Comment objects, __slots__ makes them bigger # and adding self.end did not matter attrib = comment_attrib @@ -428,6 +428,7 @@ class CommentedMap(ordereddict, CommentedBase): def __contains__(self, key): if ordereddict.__contains__(self, key): return True + # this will only work once the mapping/dict is built to completion for merged in getattr(self, merge_attrib, []): if key in merged[1]: return True @@ -439,6 +440,97 @@ class CommentedMap(ordereddict, CommentedBase): except: return default + def non_merged_items(self): + for x in ordereddict.__iter__(self): + yield x, ordereddict.__getitem__(self, x) + + def __iter__(self): + for x in ordereddict.__iter__(self): + yield x + done = [] # list of processed merge items, kept for masking + for merged in getattr(self, merge_attrib, []): + for x in merged[1]: + if ordereddict.__contains__(self, x): + continue + for y in done: + if x in y: + break + else: + yield x + done.append(merged[1]) + + def _keys(self): + for x in ordereddict.__iter__(self): + yield x + done = [] # list of processed merge items, kept for masking + for merged in getattr(self, merge_attrib, []): + for x in merged[1]: + if ordereddict.__contains__(self, x): + continue + for y in done: + if x in y: + break + else: + yield x + done.append(merged[1]) + + if PY2: + def keys(self): + return list(self._keys()) + else: + # def keys(self): + # import collections + # return collections.KeysView(self._keys()) + keys = _keys + + def _values(self): + for x in ordereddict.__iter__(self): + yield ordereddict.__getitem__(self, x) + done = [] # list of processed merge items, kept for masking + for merged in getattr(self, merge_attrib, []): + for x in merged[1]: + if ordereddict.__contains__(self, x): + continue + for y in done: + if x in y: + break + else: + yield ordereddict.__getitem__(merged[1], x) + done.append(merged[1]) + + if PY2: + def values(self): + return list(self._values()) + else: + # def values(self): + # import collections + # return collections.ValuesView(self) + values = _values + + def _items(self): + for x in ordereddict.__iter__(self): + yield x, ordereddict.__getitem__(self, x) + done = [] # list of processed merge items, kept for masking + for merged in getattr(self, merge_attrib, []): + for x in merged[1]: + if ordereddict.__contains__(self, x): + continue + for y in done: + if x in y: + break + else: + yield x, ordereddict.__getitem__(merged[1], x) + done.append(merged[1]) + + if PY2: + def items(self): + return list(self._items()) + else: + # def items(self): + # import collections + # return collections.ItemsView(self._items()) + items = _items + @property def merge(self): if not hasattr(self, merge_attrib): diff --git a/constructor.py b/constructor.py index c86989a..a07a8e3 100644 --- a/constructor.py +++ b/constructor.py @@ -941,8 +941,6 @@ class RoundTripConstructor(SafeConstructor): "expected a mapping node, but found %s" % node.id, node.start_mark) merge_map = self.flatten_mapping(node) - if merge_map: - maptyp.add_yaml_merge(merge_map) # mapping = {} if node.comment: maptyp._yaml_add_comment(node.comment[:2]) @@ -981,6 +979,10 @@ class RoundTripConstructor(SafeConstructor): key, [key_node.start_mark.line, key_node.start_mark.column, value_node.start_mark.line, value_node.start_mark.column]) maptyp[key] = value + # do this last, or <<: before a key will prevent insertion in instances + # of collections.OrderedDict (as they have no __contains__ + if merge_map: + maptyp.add_yaml_merge(merge_map) def construct_setting(self, node, typ, deep=False): if not isinstance(node, MappingNode): diff --git a/representer.py b/representer.py index 6be2e13..6129a12 100644 --- a/representer.py +++ b/representer.py @@ -689,7 +689,12 @@ class RoundTripRepresenter(SafeRepresenter): pass except AttributeError: item_comments = {} - for item_key, item_value in mapping.items(): + merge_list = [m[1] for m in getattr(mapping, merge_attrib, [])] + if merge_list: + items = mapping.non_merged_items() + else: + items = mapping.items() + for item_key, item_value in items: node_key = self.represent_key(item_key) node_value = self.represent_data(item_value) item_comment = item_comments.get(item_key) @@ -713,7 +718,6 @@ class RoundTripRepresenter(SafeRepresenter): node.flow_style = self.default_flow_style else: node.flow_style = best_style - merge_list = [m[1] for m in getattr(mapping, merge_attrib, [])] if merge_list: # because of the call to represent_data here, the anchors # are marked as being used and thereby created |