added .keys(), items(), values(), corrected __contains__

There was a huge problem with 3.4 and 3.3 as the OrderedDict
does not define a __contains__ and the implementation based on
the one by Ngo would not work with merge keys defining keys
that later were in the main dict (which should override).
Fix was to add the merge key info at the point the dict
has been completely created, thus restricting __contains__
automatically to the dict itself (and not test subkeys)

5 files changed, 168 insertions, 10 deletions

diff --git a/README.rst b/README.rst
index 8c70e81..c54aed5 100644
--- a/README.rst
+++ b/README.rst
@@ -17,9 +17,10 @@ ChangeLog
 =========
 
 ::
-
-  0.12.3 (2016-08-XX):
-    - proper 'in' operation for merged CommentedMaps (implementation by J.Ngo)
+  0.12.3 (XXXX-XX-XX):
+  - correct 'in' operation for merged CommentedMaps in round-trip mode
+   (implementation inspired by J.Ngo, but original not working for merges)
+  - iteration over round-trip loaded mappings, as well as 
 
   0.12.2 (2016-08-16):
     - minor improvements based on feedback from M. Crusoe
diff --git a/_test/test_anchor.py b/_test/test_anchor.py
index 929b1ed..1c795c0 100644
--- a/_test/test_anchor.py
+++ b/_test/test_anchor.py
@@ -175,7 +175,7 @@ class TestAnchorsAliases:
         k: &level_2 { a: 1, b2 }
         l: &level_1 { a: 10, c: 3 }
         m:
-          << : *level_1
+          <<: *level_1
           c: 30
           d: 40
         """)
@@ -227,3 +227,62 @@ class TestAnchorsAliases:
           c: 3
         b: 2
         """)
+
+
+class TestMergeKeysValues:
+
+    yaml_str = dedent("""\
+    - &mx
+      a: x1
+      b: x2
+      c: x3
+    - &my
+      a: y1
+      b: y2  # masked by the one in &mx
+      d: y4
+    -
+      a: 1
+      <<: *mx
+      m: 6
+      <<: *my
+        """)
+
+    def test_merge_for(self):
+        from ruamel.yaml import safe_load
+        d = safe_load(self.yaml_str)
+        data = round_trip_load(self.yaml_str)
+        count = 0
+        for x in data[2]:
+            count += 1
+            print(count, x)
+        assert count == len(d[2])
+
+    def test_merge_keys(self):
+        from ruamel.yaml import safe_load
+        d = safe_load(self.yaml_str)
+        data = round_trip_load(self.yaml_str)
+        count = 0
+        for x in data[2].keys():
+            count += 1
+            print(count, x)
+        assert count == len(d[2])
+
+    def test_merge_values(self):
+        from ruamel.yaml import safe_load
+        d = safe_load(self.yaml_str)
+        data = round_trip_load(self.yaml_str)
+        count = 0
+        for x in data[2].values():
+            count += 1
+            print(count, x)
+        assert count == len(d[2])
+
+    def test_merge_items(self):
+        from ruamel.yaml import safe_load
+        d = safe_load(self.yaml_str)
+        data = round_trip_load(self.yaml_str)
+        count = 0
+        for x in data[2].items():
+            count += 1
+            print(count, x)
+        assert count == len(d[2])
diff --git a/comments.py b/comments.py
index 07e2e9b..26587ce 100644
--- a/comments.py
+++ b/comments.py
@@ -10,7 +10,7 @@ a separate base
 
 from collections import MutableSet  # type: ignore
 
-from ruamel.yaml.compat import ordereddict
+from ruamel.yaml.compat import ordereddict, PY2
 
 __all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap",
            "CommentedSet", 'comment_attrib', 'merge_attrib']
@@ -24,7 +24,7 @@ tag_attrib = '_yaml_tag'
 
 
 class Comment(object):
-    # sys.getsize tested the Comment objects, __slots__ make them bigger
+    # sys.getsize tested the Comment objects, __slots__ makes them bigger
     # and adding self.end did not matter
     attrib = comment_attrib
 
@@ -428,6 +428,7 @@ class CommentedMap(ordereddict, CommentedBase):
     def __contains__(self, key):
         if ordereddict.__contains__(self, key):
             return True
+        # this will only work once the mapping/dict is built to completion
         for merged in getattr(self, merge_attrib, []):
             if key in merged[1]:
                 return True
@@ -439,6 +440,97 @@ class CommentedMap(ordereddict, CommentedBase):
         except:
             return default
 
+    def non_merged_items(self):
+        for x in ordereddict.__iter__(self):
+            yield x, ordereddict.__getitem__(self, x)
+
+    def __iter__(self):
+        for x in ordereddict.__iter__(self):
+            yield x
+        done = []  # list of processed merge items, kept for masking
+        for merged in getattr(self, merge_attrib, []):
+            for x in merged[1]:
+                if ordereddict.__contains__(self, x):
+                    continue
+                for y in done:
+                    if x in y:
+                        break
+                else:
+                    yield x
+            done.append(merged[1])
+
+    def _keys(self):
+        for x in ordereddict.__iter__(self):
+            yield x
+        done = []  # list of processed merge items, kept for masking
+        for merged in getattr(self, merge_attrib, []):
+            for x in merged[1]:
+                if ordereddict.__contains__(self, x):
+                    continue
+                for y in done:
+                    if x in y:
+                        break
+                else:
+                    yield x
+            done.append(merged[1])
+
+    if PY2:
+        def keys(self):
+            return list(self._keys())
+    else:
+        # def keys(self):
+        #     import collections
+        #     return collections.KeysView(self._keys())
+        keys = _keys
+
+    def _values(self):
+        for x in ordereddict.__iter__(self):
+            yield ordereddict.__getitem__(self, x)
+        done = []  # list of processed merge items, kept for masking
+        for merged in getattr(self, merge_attrib, []):
+            for x in merged[1]:
+                if ordereddict.__contains__(self, x):
+                    continue
+                for y in done:
+                    if x in y:
+                        break
+                else:
+                    yield ordereddict.__getitem__(merged[1], x)
+            done.append(merged[1])
+
+    if PY2:
+        def values(self):
+            return list(self._values())
+    else:
+        # def values(self):
+        #     import collections
+        #     return collections.ValuesView(self)
+        values = _values
+
+    def _items(self):
+        for x in ordereddict.__iter__(self):
+            yield x, ordereddict.__getitem__(self, x)
+        done = []  # list of processed merge items, kept for masking
+        for merged in getattr(self, merge_attrib, []):
+            for x in merged[1]:
+                if ordereddict.__contains__(self, x):
+                    continue
+                for y in done:
+                    if x in y:
+                        break
+                else:
+                    yield x, ordereddict.__getitem__(merged[1], x)
+            done.append(merged[1])
+
+    if PY2:
+        def items(self):
+            return list(self._items())
+    else:
+        # def items(self):
+        #    import collections
+        #    return collections.ItemsView(self._items())
+        items = _items
+
     @property
     def merge(self):
         if not hasattr(self, merge_attrib):
diff --git a/constructor.py b/constructor.py
index c86989a..a07a8e3 100644
--- a/constructor.py
+++ b/constructor.py
@@ -941,8 +941,6 @@ class RoundTripConstructor(SafeConstructor):
                 "expected a mapping node, but found %s" % node.id,
                 node.start_mark)
         merge_map = self.flatten_mapping(node)
-        if merge_map:
-            maptyp.add_yaml_merge(merge_map)
         # mapping = {}
         if node.comment:
             maptyp._yaml_add_comment(node.comment[:2])
@@ -981,6 +979,10 @@ class RoundTripConstructor(SafeConstructor):
                 key, [key_node.start_mark.line, key_node.start_mark.column,
                       value_node.start_mark.line, value_node.start_mark.column])
             maptyp[key] = value
+        # do this last, or <<: before a key will prevent insertion in instances
+        # of collections.OrderedDict (as they have no __contains__
+        if merge_map:
+            maptyp.add_yaml_merge(merge_map)
 
     def construct_setting(self, node, typ, deep=False):
         if not isinstance(node, MappingNode):
diff --git a/representer.py b/representer.py
index 6be2e13..6129a12 100644
--- a/representer.py
+++ b/representer.py
@@ -689,7 +689,12 @@ class RoundTripRepresenter(SafeRepresenter):
                 pass
         except AttributeError:
             item_comments = {}
-        for item_key, item_value in mapping.items():
+        merge_list = [m[1] for m in getattr(mapping, merge_attrib, [])]
+        if merge_list:
+            items = mapping.non_merged_items()
+        else:
+            items = mapping.items()
+        for item_key, item_value in items:
             node_key = self.represent_key(item_key)
             node_value = self.represent_data(item_value)
             item_comment = item_comments.get(item_key)
@@ -713,7 +718,6 @@ class RoundTripRepresenter(SafeRepresenter):
                 node.flow_style = self.default_flow_style
             else:
                 node.flow_style = best_style
-        merge_list = [m[1] for m in getattr(mapping, merge_attrib, [])]
         if merge_list:
             # because of the call to represent_data here, the anchors
             # are marked as being used and thereby created