mypy, split contruct_object0.16.3

fixes issue #306


*When this change indeed resolves your problem, please **Close** this issue*.
*(You can do so using the WorkFlow pull-down (close to the top right of this page))*

16 files changed, 78 insertions, 48 deletions

diff --git a/CHANGES b/CHANGES
index 4df6850..81dbe62 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,7 @@
+[0, 16, 3]: 2019-08-15
+  - move setting of version based on YAML directive to scanner, allowing to
+    check for file version during TAG directive scanning
+
 [0, 16, 2]: 2019-08-15
   - preserve YAML and TAG directives on roundtrip, correctly output #
     in URL for YAML 1.2 (both reported by `Thomas Smith
diff --git a/README.rst b/README.rst
index c7152a8..04f83b7 100644
--- a/README.rst
+++ b/README.rst
@@ -4,7 +4,7 @@ ruamel.yaml
 
 ``ruamel.yaml`` is a YAML 1.2 loader/dumper package for Python.
 
-:version:       0.16.2
+:version:       0.16.3
 :updated:       2019-08-15
 :documentation: http://yaml.readthedocs.io
 :repository:    https://bitbucket.org/ruamel/yaml
@@ -54,6 +54,12 @@ ChangeLog
 
 .. should insert NEXT: at the beginning of line for next key (with empty line)
 
+0.16.3 (2019-08-15):
+  - split construct_object
+  - change stuff back to keep mypy happy
+  - move setting of version based on YAML directive to scanner, allowing to
+    check for file version during TAG directive scanning
+
 0.16.2 (2019-08-15):
   - preserve YAML and TAG directives on roundtrip, correctly output #
     in URL for YAML 1.2 (both reported by `Thomas Smith
diff --git a/__init__.py b/__init__.py
index 7e1b208..fa58ebc 100644
--- a/__init__.py
+++ b/__init__.py
@@ -7,8 +7,8 @@ if False:  # MYPY
 
 _package_data = dict(
     full_package_name='ruamel.yaml',
-    version_info=(0, 16, 2),
-    __version__='0.16.2',
+    version_info=(0, 16, 3),
+    __version__='0.16.3',
     author='Anthon van der Neut',
     author_email='a.van.der.neut@ruamel.eu',
     description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order',  # NOQA
diff --git a/_doc/_static/pypi.svg b/_doc/_static/pypi.svg
index dc4abf1..7e4cfcd 100644
--- a/_doc/_static/pypi.svg
+++ b/_doc/_static/pypi.svg
@@ -1 +1 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="86" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="86" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h33v20H0z"/><path fill="#007ec6" d="M33 0h53v20H33z"/><path fill="url(#b)" d="M0 0h86v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="175" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="230">pypi</text><text x="175" y="140" transform="scale(.1)" textLength="230">pypi</text><text x="585" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">0.16.2</text><text x="585" y="140" transform="scale(.1)" textLength="430">0.16.2</text></g> </svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="86" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="86" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h33v20H0z"/><path fill="#007ec6" d="M33 0h53v20H33z"/><path fill="url(#b)" d="M0 0h86v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="175" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="230">pypi</text><text x="175" y="140" transform="scale(.1)" textLength="230">pypi</text><text x="585" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">0.16.3</text><text x="585" y="140" transform="scale(.1)" textLength="430">0.16.3</text></g> </svg>
diff --git a/comments.py b/comments.py
index e785ae7..61c1f0e 100644
--- a/comments.py
+++ b/comments.py
@@ -12,7 +12,8 @@ import sys
 import copy
 
 
-from ruamel.yaml.compat import ordereddict, PY2, string_types, MutableSliceableSequence
+from ruamel.yaml.compat import ordereddict  # type: ignore
+from ruamel.yaml.compat import PY2, string_types, MutableSliceableSequence
 from ruamel.yaml.scalarstring import ScalarString
 from ruamel.yaml.anchor import Anchor
 
@@ -159,7 +160,7 @@ class LineCol(object):
     def add_idx_line_col(self, key, data):
         # type: (Any, Any) -> None
         if self.data is None:
-            self.data = {}  # type: Dict[Any, Any]
+            self.data = {}
         self.data[key] = data
 
 
@@ -351,7 +352,7 @@ class CommentedBase(object):
         self.tag.value = value
 
     def copy_attributes(self, t, memo=None):
-        # type: (Any, bool) -> None
+        # type: (Any, Any) -> None
         # fmt: off
         for a in [Comment.attrib, Format.attrib, LineCol.attrib, Anchor.attrib,
                   Tag.attrib, merge_attrib]:
@@ -629,7 +630,7 @@ class CommentedMapValuesView(CommentedMapView):
             yield self._mapping[key]
 
 
-class CommentedMap(ordereddict, CommentedBase):
+class CommentedMap(ordereddict, CommentedBase):  # type: ignore
     __slots__ = (Comment.attrib, '_ok', '_ref')
 
     def __init__(self, *args, **kw):
@@ -695,7 +696,7 @@ class CommentedMap(ordereddict, CommentedBase):
             self.ca.comment[1] = pre_comments
         return pre_comments
 
-    def update(self, vals):  # type: ignore
+    def update(self, vals):
         # type: (Any) -> None
         try:
             ordereddict.update(self, vals)
@@ -828,7 +829,7 @@ class CommentedMap(ordereddict, CommentedBase):
 
     def __len__(self):
         # type: () -> int
-        return ordereddict.__len__(self)
+        return int(ordereddict.__len__(self))
 
     def __eq__(self, other):
         # type: (Any) -> bool
diff --git a/compat.py b/compat.py
index 5529247..839166f 100644
--- a/compat.py
+++ b/compat.py
@@ -13,7 +13,8 @@ from abc import abstractmethod
 
 # fmt: off
 if False:  # MYPY
-    from typing import Any, Dict, Optional, List, Union, BinaryIO, IO, Text, Tuple, Optional  # NOQA
+    from typing import Any, Dict, Optional, List, Union, BinaryIO, IO, Text, Tuple  # NOQA
+    from typing import Optional  # NOQA
 # fmt: on
 
 _DEFAULT_YAML_VERSION = (1, 2)
@@ -106,7 +107,7 @@ else:
     binary_type = str
 
     # to allow importing
-    unichr = unichr  # type: ignore
+    unichr = unichr
     from StringIO import StringIO as _StringIO
 
     StringIO = _StringIO
@@ -122,7 +123,7 @@ if False:  # MYPY
     # StreamType = Union[BinaryIO, IO[str], StringIO]  # type: ignore
     StreamType = Any
 
-    StreamTextType = Union[Text, StreamType]
+    StreamTextType = StreamType  # Union[Text, StreamType]
     VersionType = Union[List[int], str, Tuple[int, int]]
 
 if PY3:
diff --git a/constructor.py b/constructor.py
index ee605ea..758daca 100644
--- a/constructor.py
+++ b/constructor.py
@@ -16,8 +16,10 @@ from ruamel.yaml.error import (MarkedYAMLError, MarkedYAMLFutureWarning,
 from ruamel.yaml.nodes import *                               # NOQA
 from ruamel.yaml.nodes import (SequenceNode, MappingNode, ScalarNode)
 from ruamel.yaml.compat import (utf8, builtins_module, to_str, PY2, PY3,  # NOQA
-                                ordereddict, text_type, nprint, nprintf, version_tnf,
-                                Hashable, MutableSequence, MutableMapping)
+                                text_type, nprint, nprintf, version_tnf)
+from ruamel.yaml.compat import ordereddict, Hashable, MutableSequence  # type: ignore
+from ruamel.yaml.compat import MutableMapping  # type: ignore
+
 from ruamel.yaml.comments import *                               # NOQA
 from ruamel.yaml.comments import (CommentedMap, CommentedOrderedMap, CommentedSet,
                                   CommentedKeySeq, CommentedSeq, TaggedScalar,
@@ -141,19 +143,31 @@ class BaseConstructor(object):
             #     None, None, 'found unconstructable recursive node', node.start_mark
             # )
         self.recursive_objects[node] = None
+        data = self.construct_non_recursive_object(node)
+
+        self.constructed_objects[node] = data
+        del self.recursive_objects[node]
+        if deep:
+            self.deep_construct = old_deep
+        return data
+
+    def construct_non_recursive_object(self, node, tag=None):
+        # type: (Any, Optional[str]) -> Any
         constructor = None  # type: Any
         tag_suffix = None
-        if node.tag in self.yaml_constructors:
-            constructor = self.yaml_constructors[node.tag]
+        if tag is None:
+            tag = node.tag
+        if tag in self.yaml_constructors:
+            constructor = self.yaml_constructors[tag]
         else:
             for tag_prefix in self.yaml_multi_constructors:
-                if node.tag.startswith(tag_prefix):
-                    tag_suffix = node.tag[len(tag_prefix) :]
+                if tag.startswith(tag_prefix):
+                    tag_suffix = tag[len(tag_prefix) :]
                     constructor = self.yaml_multi_constructors[tag_prefix]
                     break
             else:
                 if None in self.yaml_multi_constructors:
-                    tag_suffix = node.tag
+                    tag_suffix = tag
                     constructor = self.yaml_multi_constructors[None]
                 elif None in self.yaml_constructors:
                     constructor = self.yaml_constructors[None]
@@ -175,10 +189,6 @@ class BaseConstructor(object):
                     pass
             else:
                 self.state_generators.append(generator)
-        self.constructed_objects[node] = data
-        del self.recursive_objects[node]
-        if deep:
-            self.deep_construct = old_deep
         return data
 
     def construct_scalar(self, node):
diff --git a/emitter.py b/emitter.py
index 1b6006f..02100dd 100644
--- a/emitter.py
+++ b/emitter.py
@@ -241,7 +241,7 @@ class Emitter(object):
     def dispose(self):
         # type: () -> None
         # Reset the state attributes (to clear self-references)
-        self.states = []  # type: List[Any]
+        self.states = []
         self.state = None
 
     def emit(self, event):
@@ -1591,7 +1591,7 @@ class Emitter(object):
                         self.stream.write(data)
                     start = end
             elif breaks:
-                if ch not in u'\n\x85\u2028\u2029':
+                if ch not in u'\n\x85\u2028\u2029':  # type: ignore
                     if text[start] == u'\n':
                         self.write_line_break()
                     for br in text[start:end]:
diff --git a/error.py b/error.py
index 965ff7b..d5f1553 100644
--- a/error.py
+++ b/error.py
@@ -40,6 +40,7 @@ class StreamMark(object):
         return where
 
     def __eq__(self, other):
+        # type: (Any) -> bool
         if self.line != other.line or self.column != other.column:
             return False
         if self.name != other.name or self.index != other.index:
@@ -47,6 +48,7 @@ class StreamMark(object):
         return True
 
     def __ne__(self, other):
+        # type: (Any) -> bool
         return not self.__eq__(other)
 
 
diff --git a/main.py b/main.py
index 2d5103e..999f957 100644
--- a/main.py
+++ b/main.py
@@ -35,7 +35,7 @@ from ruamel.yaml.constructor import (
 from ruamel.yaml.loader import Loader as UnsafeLoader
 
 if False:  # MYPY
-    from typing import List, Set, Dict, Union, Any, Callable  # NOQA
+    from typing import List, Set, Dict, Union, Any, Callable, Optional, Text  # NOQA
     from ruamel.yaml.compat import StreamType, StreamTextType, VersionType  # NOQA
 
     if PY3:
@@ -99,9 +99,9 @@ class YAML(object):
             self.default_flow_style = False
             # no optimized rt-dumper yet
             self.Emitter = ruamel.yaml.emitter.Emitter  # type: Any
-            self.Serializer = ruamel.yaml.serializer.Serializer  # type: Any
+            self.Serializer = ruamel.yaml.serializer.Serializer
             self.Representer = ruamel.yaml.representer.RoundTripRepresenter  # type: Any
-            self.Scanner = ruamel.yaml.scanner.RoundTripScanner  # type: Any
+            self.Scanner = ruamel.yaml.scanner.RoundTripScanner
             # no optimized rt-parser yet
             self.Parser = ruamel.yaml.parser.RoundTripParser  # type: Any
             self.Composer = ruamel.yaml.composer.Composer  # type: Any
@@ -326,7 +326,7 @@ class YAML(object):
         """
         if not hasattr(stream, 'read') and hasattr(stream, 'open'):
             # pathlib.Path() instance
-            with stream.open('rb') as fp:  # type: ignore
+            with stream.open('rb') as fp:
                 return self.load(fp)
         constructor, parser = self.get_constructor_parser(stream)
         try:
@@ -351,7 +351,7 @@ class YAML(object):
             )
         if not hasattr(stream, 'read') and hasattr(stream, 'open'):
             # pathlib.Path() instance
-            with stream.open('r') as fp:  # type: ignore
+            with stream.open('r') as fp:
                 for d in self.load_all(fp, _kw=enforce):
                     yield d
                 return
diff --git a/representer.py b/representer.py
index 946a909..a0078f2 100644
--- a/representer.py
+++ b/representer.py
@@ -5,7 +5,8 @@ from __future__ import print_function, absolute_import, division
 
 from ruamel.yaml.error import *  # NOQA
 from ruamel.yaml.nodes import *  # NOQA
-from ruamel.yaml.compat import text_type, binary_type, to_unicode, PY2, PY3, ordereddict
+from ruamel.yaml.compat import text_type, binary_type, to_unicode, PY2, PY3
+from ruamel.yaml.compat import ordereddict  # type: ignore
 from ruamel.yaml.compat import nprint, nprintf  # NOQA
 from ruamel.yaml.scalarstring import (
     LiteralScalarString,
@@ -83,8 +84,8 @@ class BaseRepresenter(object):
         # type: (Any) -> None
         node = self.represent_data(data)
         self.serializer.serialize(node)
-        self.represented_objects = {}  # type: Dict[Any, Any]
-        self.object_keeper = []  # type: List[Any]
+        self.represented_objects = {}
+        self.object_keeper = []
         self.alias_key = None
 
     def represent_data(self, data):
diff --git a/scalarstring.py b/scalarstring.py
index bf3cbab..8427323 100644
--- a/scalarstring.py
+++ b/scalarstring.py
@@ -129,7 +129,8 @@ def walk_tree(base, map=None):
         map[':'] = SingleQuotedScalarString
         walk_tree(data, map=map)
     """
-    from ruamel.yaml.compat import string_types, MutableMapping, MutableSequence
+    from ruamel.yaml.compat import string_types
+    from ruamel.yaml.compat import MutableMapping, MutableSequence  # type: ignore
 
     if map is None:
         map = {'\n': preserve_literal}
diff --git a/scanner.py b/scanner.py
index 69869a5..885bf22 100644
--- a/scanner.py
+++ b/scanner.py
@@ -81,7 +81,7 @@ class Scanner(object):
             self.loader._scanner = self
         self.reset_scanner()
         self.first_time = False
-        self.yaml_version = None
+        self.yaml_version = None  # type: Any
 
     @property
     def flow_level(self):
@@ -156,10 +156,10 @@ class Scanner(object):
 
     @property
     def scanner_processing_version(self):  # prefix until un-composited
-        # type: () -> VersionType
+        # type: () -> Any
         if hasattr(self.loader, 'typ'):
-            return self.loader.resolver.processing_version  # type: ignore
-        return self.loader.processing_version  # type: ignore
+            return self.loader.resolver.processing_version
+        return self.loader.processing_version
 
     # Public methods.
 
@@ -454,7 +454,7 @@ class Scanner(object):
         # Reset simple keys.
         self.remove_possible_simple_key()
         self.allow_simple_key = False
-        self.possible_simple_keys = {}  # type: Dict[Any, Any]
+        self.possible_simple_keys = {}
         # Read the token.
         mark = self.reader.get_mark()
         # Add STREAM-END.
diff --git a/setup.py b/setup.py
index 7c5ccb3..f18c143 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,6 @@
 # # header
 # coding: utf-8
-# dd: 20190807
+# dd: 20190815
 
 from __future__ import print_function, absolute_import, division, unicode_literals
 
@@ -20,8 +20,12 @@ from setuptools import setup, Extension, Distribution  # NOQA
 from setuptools.command import install_lib  # NOQA
 from setuptools.command.sdist import sdist as _sdist  # NOQA
 
-from setuptools.namespaces import Installer as NameSpaceInstaller # NOQA
-
+try:
+    from setuptools.namespaces import Installer as NameSpaceInstaller # NOQA
+except ImportError:
+    print("You should use the latest setuptools. The namespaces.py file that this setup.py"
+          " uses was added in setuptools 28.7.0 (Oct 2016)")
+    sys.exit()
 
 if __name__ != '__main__':
     raise NotImplementedError('should never include setup.py')
@@ -30,9 +34,6 @@ if __name__ != '__main__':
 
 full_package_name = None
 
-if __name__ != '__main__':
-    raise NotImplementedError('should never include setup.py')
-
 if sys.version_info < (3,):
     string_type = basestring
 else:
diff --git a/tokens.py b/tokens.py
index 84c6847..5f5a663 100644
--- a/tokens.py
+++ b/tokens.py
@@ -272,6 +272,7 @@ class CommentToken(Token):
         return 'CommentToken({})'.format(v)
 
     def __eq__(self, other):
+        # type: (Any) -> bool
         if self.start_mark != other.start_mark:
             return False
         if self.end_mark != other.end_mark:
@@ -281,4 +282,5 @@ class CommentToken(Token):
         return True
 
     def __ne__(self, other):
+        # type: (Any) -> bool
         return not self.__eq__(other)
diff --git a/util.py b/util.py
index 2412888..3eb7d76 100644
--- a/util.py
+++ b/util.py
@@ -77,9 +77,10 @@ def load_yaml_guess_indent(stream, **kw):
         return idx
 
     if isinstance(stream, text_type):
-        yaml_str = stream
+        yaml_str = stream  # type: Any
     elif isinstance(stream, binary_type):
-        yaml_str = stream.decode('utf-8')  # most likely, but the Reader checks BOM for this
+        # most likely, but the Reader checks BOM for this
+        yaml_str = stream.decode('utf-8')
     else:
         yaml_str = stream.read()
     map_indent = None