diff options
author | Claudiu Popa <pcmanticore@gmail.com> | 2018-05-23 18:36:45 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-23 18:36:45 +0200 |
commit | 2c655de34603e5a0f88b369d3e4b68556fdb0641 (patch) | |
tree | 2dc879f74c672643f37ae084bcb389e64c3ed97d | |
parent | 5d17f5b1cbb875cc2ca28cee0942f3f92fb2a01d (diff) | |
download | astroid-git-2c655de34603e5a0f88b369d3e4b68556fdb0641.tar.gz |
Add support for type comments (#548)
-rw-r--r-- | ChangeLog | 6 | ||||
-rw-r--r-- | astroid/__pkginfo__.py | 6 | ||||
-rw-r--r-- | astroid/_ast.py | 21 | ||||
-rw-r--r-- | astroid/builder.py | 7 | ||||
-rw-r--r-- | astroid/node_classes.py | 29 | ||||
-rw-r--r-- | astroid/rebuilder.py | 244 | ||||
-rw-r--r-- | astroid/tests/unittest_nodes.py | 62 | ||||
-rw-r--r-- | pylintrc | 2 |
8 files changed, 270 insertions, 107 deletions
@@ -2,6 +2,12 @@ Change log for the astroid package (used to be astng) ===================================================== -- 2.0 +-- + * Switched to using typed_ast for getting access to type comments + + As a side effect of this change, some nodes gained a new `type_annotation` attribute, + which, if the type comments were correctly parsed, should contain a node object + with the corresponding objects from the type comment. * typing.X[...] and typing.NewType are inferred as classes instead of instances. diff --git a/astroid/__pkginfo__.py b/astroid/__pkginfo__.py index c873f595..8be5ede5 100644 --- a/astroid/__pkginfo__.py +++ b/astroid/__pkginfo__.py @@ -6,6 +6,7 @@ # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER """astroid packaging information""" +import platform distname = 'astroid' @@ -19,9 +20,12 @@ install_requires = [ 'lazy_object_proxy', 'six', 'wrapt', - 'typing;python_version<"3.5"' + 'typing;python_version<"3.5"', ] +if platform.python_implementation() == 'CPython': + install_requires.append('typed_ast;python_version<"3.7"') + # pylint: disable=redefined-builtin; why license is a builtin anyway? license = 'LGPL' diff --git a/astroid/_ast.py b/astroid/_ast.py new file mode 100644 index 00000000..8220e108 --- /dev/null +++ b/astroid/_ast.py @@ -0,0 +1,21 @@ +import ast + +_ast_py2 = _ast_py3 = None +try: + import typed_ast.ast3 as _ast_py3 + import typed_ast.ast27 as _ast_py2 +except ImportError: + pass + + +def _get_parser_module(parse_python_two: bool = False): + if parse_python_two: + parser_module = _ast_py2 + else: + parser_module = _ast_py3 + return parser_module or ast + + +def _parse(string: str, + parse_python_two: bool = False): + return _get_parser_module(parse_python_two=parse_python_two).parse(string) diff --git a/astroid/builder.py b/astroid/builder.py index 2522ef01..320d23a3 100644 --- a/astroid/builder.py +++ b/astroid/builder.py @@ -16,8 +16,9 @@ import re import os import sys import textwrap -import _ast + +from astroid._ast import _parse from astroid import bases from astroid import exceptions from astroid import manager @@ -37,10 +38,6 @@ _TRANSIENT_FUNCTION = '__' _STATEMENT_SELECTOR = '#@' -def _parse(string): - return compile(string, "<string>", 'exec', _ast.PyCF_ONLY_AST) - - if sys.version_info >= (3, 0): from tokenize import detect_encoding diff --git a/astroid/node_classes.py b/astroid/node_classes.py index caa66239..53ee065e 100644 --- a/astroid/node_classes.py +++ b/astroid/node_classes.py @@ -1699,6 +1699,7 @@ class Assign(mixins.AssignTypeMixin, Statement): <Assign l.1 at 0x7effe1db8550> """ _astroid_fields = ('targets', 'value',) + _other_other_fields = ('type_annotation',) targets = None """What is being assigned to. @@ -1709,8 +1710,13 @@ class Assign(mixins.AssignTypeMixin, Statement): :type: NodeNG or None """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment - def postinit(self, targets=None, value=None): + :type: NodeNG or None + """ + + def postinit(self, targets=None, value=None, type_annotation=None): """Do some setup after initialisation. :param targets: What is being assigned to. @@ -1721,6 +1727,7 @@ class Assign(mixins.AssignTypeMixin, Statement): """ self.targets = targets self.value = value + self.type_annotation = type_annotation def get_children(self): yield from self.targets @@ -2912,6 +2919,7 @@ class For(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, <For l.1 at 0x7f23b2e8cf28> """ _astroid_fields = ('target', 'iter', 'body', 'orelse',) + _other_other_fields = ('type_annotation',) _multi_line_block_fields = ('body', 'orelse') target = None """What the loop assigns to. @@ -2933,9 +2941,14 @@ class For(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, :type: list(NodeNG) or None """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment + + :type: NodeNG or None + """ # pylint: disable=redefined-builtin; had to use the same name as builtin ast module. - def postinit(self, target=None, iter=None, body=None, orelse=None): + def postinit(self, target=None, iter=None, body=None, orelse=None, type_annotation=None): """Do some setup after initialisation. :param target: What the loop assigns to. @@ -2954,6 +2967,7 @@ class For(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, self.iter = iter self.body = body self.orelse = orelse + self.type_annotation = type_annotation optional_assign = True """Whether this node optionally assigns a variable. @@ -4231,7 +4245,8 @@ class With(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, >>> node <With l.2 at 0x7f23b2e4e710> """ - _astroid_fields = ('items', 'body') + _astroid_fields = ('items', 'body',) + _other_other_fields = ('type_annotation',) _multi_line_block_fields = ('body',) items = None """The pairs of context managers and the names they are assigned to. @@ -4243,8 +4258,13 @@ class With(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, :type: list(NodeNG) or None """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment + + :type: NodeNG or None + """ - def postinit(self, items=None, body=None): + def postinit(self, items=None, body=None, type_annotation=None): """Do some setup after initialisation. :param items: The pairs of context managers and the names @@ -4256,6 +4276,7 @@ class With(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, """ self.items = items self.body = body + self.type_annotation = type_annotation @decorators.cachedproperty def blockstart_tolineno(self): diff --git a/astroid/rebuilder.py b/astroid/rebuilder.py index e77c2c28..223ddcdd 100644 --- a/astroid/rebuilder.py +++ b/astroid/rebuilder.py @@ -11,51 +11,12 @@ order to get a single Astroid representation """ import sys -import _ast import astroid +from astroid._ast import _parse, _get_parser_module from astroid import nodes - -_BIN_OP_CLASSES = {_ast.Add: '+', - _ast.BitAnd: '&', - _ast.BitOr: '|', - _ast.BitXor: '^', - _ast.Div: '/', - _ast.FloorDiv: '//', - _ast.Mod: '%', - _ast.Mult: '*', - _ast.Pow: '**', - _ast.Sub: '-', - _ast.LShift: '<<', - _ast.RShift: '>>', - } -if sys.version_info >= (3, 5): - _BIN_OP_CLASSES[_ast.MatMult] = '@' - -_BOOL_OP_CLASSES = {_ast.And: 'and', - _ast.Or: 'or', - } - -_UNARY_OP_CLASSES = {_ast.UAdd: '+', - _ast.USub: '-', - _ast.Not: 'not', - _ast.Invert: '~', - } - -_CMP_OP_CLASSES = {_ast.Eq: '==', - _ast.Gt: '>', - _ast.GtE: '>=', - _ast.In: 'in', - _ast.Is: 'is', - _ast.IsNot: 'is not', - _ast.Lt: '<', - _ast.LtE: '<=', - _ast.NotEq: '!=', - _ast.NotIn: 'not in', - } - CONST_NAME_TRANSFORMS = {'None': None, 'True': True, 'False': False, @@ -71,27 +32,67 @@ REDIRECT = {'arguments': 'Arguments', PY3 = sys.version_info >= (3, 0) PY34 = sys.version_info >= (3, 4) PY37 = sys.version_info >= (3, 7) -CONTEXTS = {_ast.Load: astroid.Load, - _ast.Store: astroid.Store, - _ast.Del: astroid.Del, - _ast.Param: astroid.Store} - - -def _get_doc(node): - - try: - if PY37 and hasattr(node, 'docstring'): - doc = node.docstring - return node, doc - elif (node.body - and isinstance(node.body[0], _ast.Expr) - and isinstance(node.body[0].value, _ast.Str)): - doc = node.body[0].value.s - node.body = node.body[1:] - return node, doc - except IndexError: - pass # ast built from scratch - return node, None + + +def _binary_operators_from_module(module): + binary_operators = { + module.Add: '+', + module.BitAnd: '&', + module.BitOr: '|', + module.BitXor: '^', + module.Div: '/', + module.FloorDiv: '//', + module.Mod: '%', + module.Mult: '*', + module.Pow: '**', + module.Sub: '-', + module.LShift: '<<', + module.RShift: '>>', + } + if sys.version_info >= (3, 5): + binary_operators[module.MatMult] = '@' + return binary_operators + + +def _bool_operators_from_module(module): + return { + module.And: 'and', + module.Or: 'or', + } + + +def _unary_operators_from_module(module): + return { + module.UAdd: '+', + module.USub: '-', + module.Not: 'not', + module.Invert: '~', + } + + +def _compare_operators_from_module(module): + return { + module.Eq: '==', + module.Gt: '>', + module.GtE: '>=', + module.In: 'in', + module.Is: 'is', + module.IsNot: 'is not', + module.Lt: '<', + module.LtE: '<=', + module.NotEq: '!=', + module.NotIn: 'not in', + } + + +def _contexts_from_module(module): + return { + module.Load: astroid.Load, + module.Store: astroid.Store, + module.Del: astroid.Del, + module.Param: astroid.Store, + } + def _visit_or_none(node, attr, visitor, parent, visit='visit', **kws): @@ -106,23 +107,45 @@ def _visit_or_none(node, attr, visitor, parent, visit='visit', return None -def _get_context(node): - return CONTEXTS.get(type(node.ctx), astroid.Load) - - class TreeRebuilder(object): """Rebuilds the _ast tree to become an Astroid tree""" - def __init__(self, manager): + def __init__(self, manager, parse_python_two: bool = False): self._manager = manager self._global_names = [] self._import_from_nodes = [] self._delayed_assattr = [] self._visit_meths = {} + # Configure the right classes for the right module + self._parser_module = _get_parser_module(parse_python_two=parse_python_two) + self._unary_op_classes = _unary_operators_from_module(self._parser_module) + self._cmp_op_classes = _compare_operators_from_module(self._parser_module) + self._bool_op_classes = _bool_operators_from_module(self._parser_module) + self._bin_op_classes = _binary_operators_from_module(self._parser_module) + self._context_classes = _contexts_from_module(self._parser_module) + + def _get_doc(self, node): + try: + if PY37 and hasattr(node, 'docstring'): + doc = node.docstring + return node, doc + elif (node.body + and isinstance(node.body[0], self._parser_module.Expr) + and isinstance(node.body[0].value, self._parser_module.Str)): + doc = node.body[0].value.s + node.body = node.body[1:] + return node, doc + except IndexError: + pass # ast built from scratch + return node, None + + def _get_context(self, node): + return self._context_classes.get(type(node.ctx), astroid.Load) + def visit_module(self, node, modname, modpath, package): """visit a Module node by returning a fresh instance of it""" - node, doc = _get_doc(node) + node, doc = self._get_doc(node) newnode = nodes.Module(name=modname, doc=doc, file=modpath, path=[modpath], package=package, parent=None) @@ -219,12 +242,32 @@ class TreeRebuilder(object): newnode.postinit(self.visit(node.test, newnode), msg) return newnode + def check_type_comment(self, node): + type_comment = getattr(node, 'type_comment', None) + if not type_comment: + return None + + try: + type_comment_ast = _parse(type_comment) + except SyntaxError: + # Invalid type comment, just skip it. + return None + + type_object = self.visit(type_comment_ast.body[0], node) + if not isinstance(type_object, nodes.Expr): + return None + + return type_object.value + def visit_assign(self, node, parent): """visit a Assign node by returning a fresh instance of it""" + type_annotation = self.check_type_comment(node) newnode = nodes.Assign(node.lineno, node.col_offset, parent) - newnode.postinit([self.visit(child, newnode) - for child in node.targets], - self.visit(node.value, newnode)) + newnode.postinit( + targets=[self.visit(child, newnode) for child in node.targets], + value=self.visit(node.value, newnode), + type_annotation=type_annotation, + ) return newnode def visit_assignname(self, node, parent, node_name=None): @@ -236,7 +279,7 @@ class TreeRebuilder(object): def visit_augassign(self, node, parent): """visit a AugAssign node by returning a fresh instance of it""" - newnode = nodes.AugAssign(_BIN_OP_CLASSES[type(node.op)] + "=", + newnode = nodes.AugAssign(self._bin_op_classes[type(node.op)] + "=", node.lineno, node.col_offset, parent) newnode.postinit(self.visit(node.target, newnode), self.visit(node.value, newnode)) @@ -250,7 +293,7 @@ class TreeRebuilder(object): def visit_binop(self, node, parent): """visit a BinOp node by returning a fresh instance of it""" - newnode = nodes.BinOp(_BIN_OP_CLASSES[type(node.op)], + newnode = nodes.BinOp(self._bin_op_classes[type(node.op)], node.lineno, node.col_offset, parent) newnode.postinit(self.visit(node.left, newnode), self.visit(node.right, newnode)) @@ -258,7 +301,7 @@ class TreeRebuilder(object): def visit_boolop(self, node, parent): """visit a BoolOp node by returning a fresh instance of it""" - newnode = nodes.BoolOp(_BOOL_OP_CLASSES[type(node.op)], + newnode = nodes.BoolOp(self._bool_op_classes[type(node.op)], node.lineno, node.col_offset, parent) newnode.postinit([self.visit(child, newnode) for child in node.values]) @@ -305,7 +348,7 @@ class TreeRebuilder(object): def visit_classdef(self, node, parent, newstyle=None): """visit a ClassDef node to become astroid""" - node, doc = _get_doc(node) + node, doc = self._get_doc(node) newnode = nodes.ClassDef(node.name, doc, node.lineno, node.col_offset, parent) metaclass = None @@ -343,7 +386,7 @@ class TreeRebuilder(object): """visit a Compare node by returning a fresh instance of it""" newnode = nodes.Compare(node.lineno, node.col_offset, parent) newnode.postinit(self.visit(node.left, newnode), - [(_CMP_OP_CLASSES[op.__class__], + [(self._cmp_op_classes[op.__class__], self.visit(expr, newnode)) for (op, expr) in zip(node.ops, node.comparators)]) return newnode @@ -448,12 +491,14 @@ class TreeRebuilder(object): def _visit_for(self, cls, node, parent): """visit a For node by returning a fresh instance of it""" newnode = cls(node.lineno, node.col_offset, parent) - newnode.postinit(self.visit(node.target, newnode), - self.visit(node.iter, newnode), - [self.visit(child, newnode) - for child in node.body], - [self.visit(child, newnode) - for child in node.orelse]) + type_annotation = self.check_type_comment(node) + newnode.postinit( + target=self.visit(node.target, newnode), + iter=self.visit(node.iter, newnode), + body=[self.visit(child, newnode) for child in node.body], + orelse=[self.visit(child, newnode) for child in node.orelse], + type_annotation=type_annotation, + ) return newnode def visit_for(self, node, parent): @@ -472,7 +517,7 @@ class TreeRebuilder(object): def _visit_functiondef(self, cls, node, parent): """visit an FunctionDef node to become astroid""" self._global_names.append({}) - node, doc = _get_doc(node) + node, doc = self._get_doc(node) newnode = cls(node.name, doc, node.lineno, node.col_offset, parent) if node.decorator_list: @@ -503,7 +548,7 @@ class TreeRebuilder(object): def visit_attribute(self, node, parent): """visit an Attribute node by returning a fresh instance of it""" - context = _get_context(node) + context = self._get_context(node) if context == astroid.Del: # FIXME : maybe we should reintroduce and visit_delattr ? # for instance, deactivating assign_ctx @@ -580,7 +625,7 @@ class TreeRebuilder(object): def visit_list(self, node, parent): """visit a List node by returning a fresh instance of it""" - context = _get_context(node) + context = self._get_context(node) newnode = nodes.List(ctx=context, lineno=node.lineno, col_offset=node.col_offset, @@ -599,7 +644,7 @@ class TreeRebuilder(object): def visit_name(self, node, parent): """visit a Name node by returning a fresh instance of it""" - context = _get_context(node) + context = self._get_context(node) # True and False can be assigned to something in py2x, so we have to # check first the context. if context == astroid.Del: @@ -684,7 +729,7 @@ class TreeRebuilder(object): def visit_subscript(self, node, parent): """visit a Subscript node by returning a fresh instance of it""" - context = _get_context(node) + context = self._get_context(node) newnode = nodes.Subscript(ctx=context, lineno=node.lineno, col_offset=node.col_offset, @@ -715,7 +760,7 @@ class TreeRebuilder(object): def visit_tuple(self, node, parent): """visit a Tuple node by returning a fresh instance of it""" - context = _get_context(node) + context = self._get_context(node) newnode = nodes.Tuple(ctx=context, lineno=node.lineno, col_offset=node.col_offset, @@ -726,7 +771,7 @@ class TreeRebuilder(object): def visit_unaryop(self, node, parent): """visit a UnaryOp node by returning a fresh instance of it""" - newnode = nodes.UnaryOp(_UNARY_OP_CLASSES[node.op.__class__], + newnode = nodes.UnaryOp(self._unary_op_classes[node.op.__class__], node.lineno, node.col_offset, parent) newnode.postinit(self.visit(node.operand, newnode)) return newnode @@ -748,9 +793,13 @@ class TreeRebuilder(object): optional_vars = self.visit(node.optional_vars, newnode) else: optional_vars = None - newnode.postinit([(expr, optional_vars)], - [self.visit(child, newnode) - for child in node.body]) + + type_annotation = self.check_type_comment(node) + newnode.postinit( + items=[(expr, optional_vars)], + body=[self.visit(child, newnode) for child in node.body], + type_annotation=type_annotation, + ) return newnode def visit_yield(self, node, parent): @@ -792,7 +841,6 @@ class TreeRebuilder3(TreeRebuilder): return nodes.Nonlocal(node.names, getattr(node, 'lineno', None), getattr(node, 'col_offset', None), parent) - def visit_raise(self, node, parent): """visit a Raise node by returning a fresh instance of it""" newnode = nodes.Raise(node.lineno, node.col_offset, parent) @@ -803,7 +851,7 @@ class TreeRebuilder3(TreeRebuilder): def visit_starred(self, node, parent): """visit a Starred node and return a new instance of it""" - context = _get_context(node) + context = self._get_context(node) newnode = nodes.Starred(ctx=context, lineno=node.lineno, col_offset=node.col_offset, parent=parent) @@ -848,9 +896,13 @@ class TreeRebuilder3(TreeRebuilder): expr = self.visit(child.context_expr, newnode) var = _visit_or_none(child, 'optional_vars', self, newnode) return expr, var - newnode.postinit([visit_child(child) for child in node.items], - [self.visit(child, newnode) - for child in node.body]) + + type_annotation = self.check_type_comment(node) + newnode.postinit( + items=[visit_child(child) for child in node.items], + body=[self.visit(child, newnode) for child in node.body], + type_annotation=type_annotation, + ) return newnode def visit_with(self, node, parent): diff --git a/astroid/tests/unittest_nodes.py b/astroid/tests/unittest_nodes.py index 1963c0ce..96da3895 100644 --- a/astroid/tests/unittest_nodes.py +++ b/astroid/tests/unittest_nodes.py @@ -10,11 +10,13 @@ """tests for specific behaviour of astroid nodes """ import os +import platform import sys import textwrap import unittest import warnings +import pytest import six import astroid @@ -33,6 +35,10 @@ from astroid.tests import resources abuilder = builder.AstroidBuilder() BUILTINS = six.moves.builtins.__name__ +HAS_TYPED_AST = ( + platform.python_implementation() != 'CPython' + and sys.version_info.minor < 7 +) class AsStringTest(resources.SysPathSetup, unittest.TestCase): @@ -865,5 +871,61 @@ def test_unknown(): assert isinstance(nodes.Unknown().qname(), str) +@pytest.mark.skipif(not HAS_TYPED_AST, reason="requires typed_ast") +def test_type_comments_with(): + module = builder.parse(''' + with a as b: # type: int + pass + with a as b: # type: ignore + pass + ''') + node = module.body[0] + ignored_node = module.body[1] + assert isinstance(node.type_annotation, astroid.Name) + + assert ignored_node.type_annotation is None + + +@pytest.mark.skipif(not HAS_TYPED_AST, reason="requires typed_ast") +def test_type_comments_for(): + module = builder.parse(''' + for a, b in [1, 2, 3]: # type: List[int] + pass + for a, b in [1, 2, 3]: # type: ignore + pass + ''') + node = module.body[0] + ignored_node = module.body[1] + assert isinstance(node.type_annotation, astroid.Subscript) + assert node.type_annotation.as_string() == 'List[int]' + + assert ignored_node.type_annotation is None + + +@pytest.mark.skipif(not HAS_TYPED_AST, reason="requires typed_ast") +def test_type_coments_assign(): + module = builder.parse(''' + a, b = [1, 2, 3] # type: List[int] + a, b = [1, 2, 3] # type: ignore + ''') + node = module.body[0] + ignored_node = module.body[1] + assert isinstance(node.type_annotation, astroid.Subscript) + assert node.type_annotation.as_string() == 'List[int]' + + assert ignored_node.type_annotation is None + + +@pytest.mark.skipif(not HAS_TYPED_AST, reason="requires typed_ast") +def test_type_comments_invalid_expression(): + module = builder.parse(''' + a, b = [1, 2, 3] # type: something completely invalid + a, b = [1, 2, 3] # typeee: 2*+4 + a, b = [1, 2, 3] # type: List[int + ''') + for node in module.body: + assert node.type_annotation is None + + if __name__ == '__main__': unittest.main() @@ -281,7 +281,7 @@ ignore-mixin-members=yes # List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis
-ignored-modules=
+ignored-modules=typed_ast.ast3
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set).
|