diff options
author | hippo91 <guillaume.peillex@gmail.com> | 2021-02-07 16:41:48 +0100 |
---|---|---|
committer | hippo91 <guillaume.peillex@gmail.com> | 2021-02-07 16:41:48 +0100 |
commit | 19a40bb84bb9ba27d55d26b9e006ba2fe50a0206 (patch) | |
tree | afca46ee9a4b30effa5a6d36657b64b7f492c651 | |
parent | 9786a97e0cb1cca819784e7f2b431047867290b6 (diff) | |
parent | d60eb5e207427599fe2f4a8d44e4cb0bebcf3c9f (diff) | |
download | astroid-git-19a40bb84bb9ba27d55d26b9e006ba2fe50a0206.tar.gz |
Merge branch 'master' of https://github.com/dgilman/astroid into dgilman-master
-rw-r--r-- | ChangeLog | 39 | ||||
-rw-r--r-- | astroid/brain/brain_builtin_inference.py | 2 | ||||
-rw-r--r-- | astroid/brain/brain_collections.py | 8 | ||||
-rw-r--r-- | astroid/brain/brain_numpy_core_umath.py | 2 | ||||
-rw-r--r-- | astroid/brain/brain_six.py | 39 | ||||
-rw-r--r-- | astroid/brain/brain_subprocess.py | 7 | ||||
-rw-r--r-- | astroid/brain/brain_type.py | 64 | ||||
-rw-r--r-- | astroid/context.py | 10 | ||||
-rw-r--r-- | astroid/node_classes.py | 5 | ||||
-rw-r--r-- | astroid/rebuilder.py | 8 | ||||
-rw-r--r-- | tests/unittest_brain.py | 55 | ||||
-rw-r--r-- | tests/unittest_brain_numpy_core_umath.py | 8 | ||||
-rw-r--r-- | tests/unittest_inference.py | 103 | ||||
-rw-r--r-- | tests/unittest_nodes.py | 24 | ||||
-rw-r--r-- | tests/unittest_regrtest.py | 2 |
15 files changed, 360 insertions, 16 deletions
@@ -2,7 +2,6 @@ astroid's ChangeLog =================== - What's New in astroid 2.5.0? ============================ Release Date: TBA @@ -13,6 +12,23 @@ Release Date: TBA Fixes PyCQA/astroid#863 +* Enrich the ``brain_collection`` module so that ``__class_getitem__`` method is added to `deque` for + ``python`` version above 3.9. + +* The ``context.path`` is now a ``dict`` and the ``context.push`` method + returns ``True`` if the node has been visited a certain amount of times. + + Close #669 + +* Adds a brain for type object so that it is possible to write `type[int]` in annotation. + + Fixes PyCQA/pylint#4001 + +* Add ``__class_getitem__`` method to ``subprocess.Popen`` brain under Python 3.9 so that it is seen as subscriptable by pylint. + + Fixes PyCQA/pylint#4034 + + * Adds `degrees`, `radians`, which are `numpy ufunc` functions, in the `numpy` brain. Adds `random` function in the `numpy.random` brain. Fixes PyCQA/pylint#3856 @@ -21,6 +37,10 @@ Release Date: TBA Closes #703 +* Fix a crash in inference caused by `Uninferable` container elements + + Close #866 + * Add `python 3.9` support. * The flat attribute of ``numpy.ndarray`` is now inferred as an ``numpy.ndarray`` itself. @@ -81,6 +101,10 @@ Release Date: TBA Fixes #843 +* Fix interpretation of ``six.with_metaclass`` class definitions. + + Fixes #713 + * Reduce memory usage of astroid's module cache. * Remove dependency on `imp`. @@ -100,6 +124,19 @@ Release Date: TBA Close PyCQA/pylint#3686 +* ``is_generator`` correctly considers `Yield` nodes in `AugAssign` nodes + + This fixes a false positive with the `assignment-from-no-return` pylint check. + + Close PyCQA/pylint#3904 + +* Corrected the parent of function type comment nodes. + + These nodes used to be parented to their original ast.FunctionDef parent + but are now correctly parented to their astroid.FunctionDef parent. + + Close PyCQA/astroid#851 + What's New in astroid 2.4.2? ============================ diff --git a/astroid/brain/brain_builtin_inference.py b/astroid/brain/brain_builtin_inference.py index 0008244c..b7659cc9 100644 --- a/astroid/brain/brain_builtin_inference.py +++ b/astroid/brain/brain_builtin_inference.py @@ -223,6 +223,8 @@ def _container_generic_transform(arg, context, klass, iterables, build_elts): # TODO: Does not handle deduplication for sets. elts = [] for element in arg.elts: + if not element: + continue inferred = helpers.safe_infer(element, context=context) if inferred: evaluated_object = nodes.EvaluatedObject( diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 6594e0c7..229969c5 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -4,6 +4,7 @@ # Copyright (c) 2017 Derek Gustafson <degustaf@gmail.com> # Copyright (c) 2018 Ioana Tagirta <ioana.tagirta@gmail.com> # Copyright (c) 2019 Hugo van Kemenade <hugovk@users.noreply.github.com> +# Copyright (c) 2021 Julien Palard <julien@palard.fr> # Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html # For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER @@ -12,6 +13,9 @@ import sys import astroid +PY39 = sys.version_info >= (3, 9) + + def _collections_transform(): return astroid.parse( """ @@ -61,6 +65,10 @@ def _deque_mock(): def __mul__(self, other): pass def __imul__(self, other): pass def __rmul__(self, other): pass""" + if PY39: + base_deque_class += """ + @classmethod + def __class_getitem__(self, item): pass""" return base_deque_class diff --git a/astroid/brain/brain_numpy_core_umath.py b/astroid/brain/brain_numpy_core_umath.py index 1955e80e..73613b86 100644 --- a/astroid/brain/brain_numpy_core_umath.py +++ b/astroid/brain/brain_numpy_core_umath.py @@ -106,6 +106,7 @@ def numpy_core_umath_transform(): trunc = FakeUfuncOneArg() # Two args functions with optional kwargs + add = FakeUfuncTwoArgs() bitwise_and = FakeUfuncTwoArgs() bitwise_or = FakeUfuncTwoArgs() bitwise_xor = FakeUfuncTwoArgs() @@ -133,6 +134,7 @@ def numpy_core_umath_transform(): logical_xor = FakeUfuncTwoArgs() maximum = FakeUfuncTwoArgs() minimum = FakeUfuncTwoArgs() + multiply = FakeUfuncTwoArgs() nextafter = FakeUfuncTwoArgs() not_equal = FakeUfuncTwoArgs() power = FakeUfuncTwoArgs() diff --git a/astroid/brain/brain_six.py b/astroid/brain/brain_six.py index 389037f2..a998213f 100644 --- a/astroid/brain/brain_six.py +++ b/astroid/brain/brain_six.py @@ -22,6 +22,7 @@ from astroid import nodes SIX_ADD_METACLASS = "six.add_metaclass" +SIX_WITH_METACLASS = "six.with_metaclass" def _indent(text, prefix, predicate=None): @@ -190,6 +191,39 @@ def transform_six_add_metaclass(node): return node +def _looks_like_nested_from_six_with_metaclass(node): + if len(node.bases) != 1: + return False + base = node.bases[0] + if not isinstance(base, nodes.Call): + return False + try: + if hasattr(base.func, "expr"): + # format when explicit 'six.with_metaclass' is used + mod = base.func.expr.name + func = base.func.attrname + func = "{}.{}".format(mod, func) + else: + # format when 'with_metaclass' is used directly (local import from six) + # check reference module to avoid 'with_metaclass' name clashes + mod = base.parent.parent + import_from = mod.locals["with_metaclass"][0] + func = "{}.{}".format(import_from.modname, base.func.name) + except (AttributeError, KeyError, IndexError): + return False + return func == SIX_WITH_METACLASS + + +def transform_six_with_metaclass(node): + """Check if the given class node is defined with *six.with_metaclass* + + If so, inject its argument as the metaclass of the underlying class. + """ + call = node.bases[0] + node._metaclass = call.args[0] + node.bases = call.args[1:] + + register_module_extender(MANAGER, "six", six_moves_transform) register_module_extender( MANAGER, "requests.packages.urllib3.packages.six", six_moves_transform @@ -200,3 +234,8 @@ MANAGER.register_transform( transform_six_add_metaclass, _looks_like_decorated_with_six_add_metaclass, ) +MANAGER.register_transform( + nodes.ClassDef, + transform_six_with_metaclass, + _looks_like_nested_from_six_with_metaclass, +) diff --git a/astroid/brain/brain_subprocess.py b/astroid/brain/brain_subprocess.py index bc35704f..c19b32b1 100644 --- a/astroid/brain/brain_subprocess.py +++ b/astroid/brain/brain_subprocess.py @@ -14,6 +14,7 @@ import textwrap import astroid +PY39 = sys.version_info >= (3, 9) PY37 = sys.version_info >= (3, 7) PY36 = sys.version_info >= (3, 6) @@ -147,6 +148,12 @@ def _subprocess_transform(): "py3_args": py3_args, } ) + if PY39: + code += """ + @classmethod + def __class_getitem__(cls, item): + pass + """ init_lines = textwrap.dedent(init).splitlines() indented_init = "\n".join(" " * 4 + line for line in init_lines) diff --git a/astroid/brain/brain_type.py b/astroid/brain/brain_type.py new file mode 100644 index 00000000..4e82813f --- /dev/null +++ b/astroid/brain/brain_type.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +""" +Astroid hooks for type support. + +Starting from python3.9, type object behaves as it had __class_getitem__ method. +However it was not possible to simply add this method inside type's body, otherwise +all types would also have this method. In this case it would have been possible +to write str[int]. +Guido Van Rossum proposed a hack to handle this in the interpreter: +https://github.com/python/cpython/blob/master/Objects/abstract.c#L186-L189 + +This brain follows the same logic. It is no wise to add permanently the __class_getitem__ method +to the type object. Instead we choose to add it only in the case of a subscript node +which inside name node is type. +Doing this type[int] is allowed whereas str[int] is not. + +Thanks to Lukasz Langa for fruitful discussion. +""" +import sys + +from astroid import MANAGER, extract_node, inference_tip, nodes + + +PY39 = sys.version_info >= (3, 9) + + +def _looks_like_type_subscript(node): + """ + Try to figure out if a Name node is used inside a type related subscript + + :param node: node to check + :type node: astroid.node_classes.NodeNG + :return: true if the node is a Name node inside a type related subscript + :rtype: bool + """ + if isinstance(node, nodes.Name) and isinstance(node.parent, nodes.Subscript): + return node.name == "type" + return False + + +def infer_type_sub(node, context=None): + """ + Infer a type[...] subscript + + :param node: node to infer + :type node: astroid.node_classes.NodeNG + :param context: inference context + :type context: astroid.context.InferenceContext + :return: the inferred node + :rtype: nodes.NodeNG + """ + class_src = """ + class type: + def __class_getitem__(cls, key): + return cls + """ + node = extract_node(class_src) + return node.infer(context=context) + + +if PY39: + MANAGER.register_transform( + nodes.Name, inference_tip(infer_type_sub), _looks_like_type_subscript + ) diff --git a/astroid/context.py b/astroid/context.py index f1e06974..4bda945f 100644 --- a/astroid/context.py +++ b/astroid/context.py @@ -29,8 +29,10 @@ class InferenceContext: "extra_context", ) + maximum_path_visit = 3 + def __init__(self, path=None, inferred=None): - self.path = path or set() + self.path = path or dict() """ :type: set(tuple(NodeNG, optional(str))) @@ -87,10 +89,10 @@ class InferenceContext: Allows one to see if the given node has already been looked at for this inference context""" name = self.lookupname - if (node, name) in self.path: + if self.path.get((node, name), 0) >= self.maximum_path_visit: return True - self.path.add((node, name)) + self.path[(node, name)] = self.path.setdefault((node, name), 0) + 1 return False def clone(self): @@ -108,7 +110,7 @@ class InferenceContext: @contextlib.contextmanager def restore_path(self): - path = set(self.path) + path = dict(self.path) yield self.path = path diff --git a/astroid/node_classes.py b/astroid/node_classes.py index 86529e5a..62438e62 100644 --- a/astroid/node_classes.py +++ b/astroid/node_classes.py @@ -2125,6 +2125,11 @@ class AugAssign(mixins.AssignTypeMixin, Statement): yield self.target yield self.value + def _get_yield_nodes_skip_lambdas(self): + """An AugAssign node can contain a Yield node in the value""" + yield from self.value._get_yield_nodes_skip_lambdas() + yield from super()._get_yield_nodes_skip_lambdas() + class Repr(NodeNG): """Class representing an :class:`ast.Repr` node. diff --git a/astroid/rebuilder.py b/astroid/rebuilder.py index 3fc1a83f..e56abbf8 100644 --- a/astroid/rebuilder.py +++ b/astroid/rebuilder.py @@ -238,7 +238,7 @@ class TreeRebuilder: return type_object.value - def check_function_type_comment(self, node): + def check_function_type_comment(self, node, parent): type_comment = getattr(node, "type_comment", None) if not type_comment: return None @@ -251,10 +251,10 @@ class TreeRebuilder: returns = None argtypes = [ - self.visit(elem, node) for elem in (type_comment_ast.argtypes or []) + self.visit(elem, parent) for elem in (type_comment_ast.argtypes or []) ] if type_comment_ast.returns: - returns = self.visit(type_comment_ast.returns, node) + returns = self.visit(type_comment_ast.returns, parent) return returns, argtypes @@ -615,7 +615,7 @@ class TreeRebuilder: returns = None type_comment_args = type_comment_returns = None - type_comment_annotation = self.check_function_type_comment(node) + type_comment_annotation = self.check_function_type_comment(node, newnode) if type_comment_annotation: type_comment_returns, type_comment_args = type_comment_annotation newnode.postinit( diff --git a/tests/unittest_brain.py b/tests/unittest_brain.py index 628e6e2c..cb8f9e26 100644 --- a/tests/unittest_brain.py +++ b/tests/unittest_brain.py @@ -146,6 +146,17 @@ class CollectionsDequeTests(unittest.TestCase): self.assertIn("insert", inferred.locals) self.assertIn("index", inferred.locals) + @test_utils.require_version(maxver="3.8") + def test_deque_not_py39methods(self): + inferred = self._inferred_queue_instance() + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + inferred.getattr("__class_getitem__") + + @test_utils.require_version(minver="3.9") + def test_deque_py39methods(self): + inferred = self._inferred_queue_instance() + self.assertTrue(inferred.getattr("__class_getitem__")) + class OrderedDictTest(unittest.TestCase): def _inferred_ordered_dict_instance(self): @@ -947,6 +958,43 @@ class IOBrainTest(unittest.TestCase): self.assertEqual(raw.name, "FileIO") +@test_utils.require_version("3.9") +class TypeBrain(unittest.TestCase): + def test_type_subscript(self): + """ + Check that type object has the __class_getitem__ method + when it is used as a subscript + """ + src = builder.extract_node( + """ + a: type[int] = int + """ + ) + val_inf = src.annotation.value.inferred()[0] + self.assertIsInstance(val_inf, astroid.ClassDef) + self.assertEqual(val_inf.name, "type") + meth_inf = val_inf.getattr("__class_getitem__")[0] + self.assertIsInstance(meth_inf, astroid.FunctionDef) + + def test_invalid_type_subscript(self): + """ + Check that a type (str for example) that inherits + from type does not have __class_getitem__ method even + when it is used as a subscript + """ + src = builder.extract_node( + """ + a: str[int] = "abc" + """ + ) + val_inf = src.annotation.value.inferred()[0] + self.assertIsInstance(val_inf, astroid.ClassDef) + self.assertEqual(val_inf.name, "str") + with self.assertRaises(astroid.exceptions.AttributeInferenceError): + meth_inf = val_inf.getattr("__class_getitem__")[0] + + +@test_utils.require_version("3.6") class TypingBrain(unittest.TestCase): def test_namedtuple_base(self): klass = builder.extract_node( @@ -1298,6 +1346,13 @@ class SubprocessTest(unittest.TestCase): assert isinstance(inferred, astroid.Const) assert isinstance(inferred.value, (str, bytes)) + @test_utils.require_version("3.9") + def test_popen_does_not_have_class_getitem(self): + code = """import subprocess; subprocess.Popen""" + node = astroid.extract_node(code) + inferred = next(node.infer()) + assert "__class_getitem__" in inferred + class TestIsinstanceInference: """Test isinstance builtin inference""" diff --git a/tests/unittest_brain_numpy_core_umath.py b/tests/unittest_brain_numpy_core_umath.py index 2d2abdbe..acfaeb70 100644 --- a/tests/unittest_brain_numpy_core_umath.py +++ b/tests/unittest_brain_numpy_core_umath.py @@ -65,6 +65,7 @@ class NumpyBrainCoreUmathTest(unittest.TestCase): ) two_args_ufunc = ( + "add", "bitwise_and", "bitwise_or", "bitwise_xor", @@ -92,6 +93,7 @@ class NumpyBrainCoreUmathTest(unittest.TestCase): "logical_xor", "maximum", "minimum", + "multiply", "nextafter", "not_equal", "power", @@ -224,11 +226,9 @@ class NumpyBrainCoreUmathTest(unittest.TestCase): with self.subTest(typ=func_): inferred_values = list(self._inferred_numpy_func_call(func_)) self.assertTrue( - len(inferred_values) == 1 - or len(inferred_values) == 2 - and inferred_values[-1].pytype() is util.Uninferable, + len(inferred_values) == 1, msg="Too much inferred values ({}) for {:s}".format( - inferred_values[-1].pytype(), func_ + inferred_values, func_ ), ) self.assertTrue( diff --git a/tests/unittest_inference.py b/tests/unittest_inference.py index df9d2781..e580ee2a 100644 --- a/tests/unittest_inference.py +++ b/tests/unittest_inference.py @@ -1299,7 +1299,7 @@ class InferenceTest(resources.SysPathSetup, unittest.TestCase): result = node.inferred() assert len(result) == 2 assert isinstance(result[0], nodes.Dict) - assert result[1] is util.Uninferable + assert isinstance(result[1], nodes.Dict) def test_python25_no_relative_import(self): ast = resources.build_file("data/package/absimport.py") @@ -2982,6 +2982,23 @@ class InferenceTest(resources.SysPathSetup, unittest.TestCase): self.assertIsInstance(inferred, nodes.Const) self.assertEqual(inferred.value, 24) + def test_with_metaclass__getitem__(self): + ast_node = extract_node( + """ + class Meta(type): + def __getitem__(cls, arg): + return 24 + import six + class A(six.with_metaclass(Meta)): + pass + + A['Awesome'] #@ + """ + ) + inferred = next(ast_node.infer()) + self.assertIsInstance(inferred, nodes.Const) + self.assertEqual(inferred.value, 24) + def test_bin_op_classes(self): ast_node = extract_node( """ @@ -2998,6 +3015,23 @@ class InferenceTest(resources.SysPathSetup, unittest.TestCase): self.assertIsInstance(inferred, nodes.Const) self.assertEqual(inferred.value, 24) + def test_bin_op_classes_with_metaclass(self): + ast_node = extract_node( + """ + class Meta(type): + def __or__(self, other): + return 24 + import six + class A(six.with_metaclass(Meta)): + pass + + A | A + """ + ) + inferred = next(ast_node.infer()) + self.assertIsInstance(inferred, nodes.Const) + self.assertEqual(inferred.value, 24) + def test_bin_op_supertype_more_complicated_example(self): ast_node = extract_node( """ @@ -3335,6 +3369,22 @@ class InferenceTest(resources.SysPathSetup, unittest.TestCase): self.assertIsInstance(inferred, nodes.Const) self.assertEqual(inferred.value, 42) + def test_unary_op_classes_with_metaclass(self): + ast_node = extract_node( + """ + import six + class Meta(type): + def __invert__(self): + return 42 + class A(six.with_metaclass(Meta)): + pass + ~A + """ + ) + inferred = next(ast_node.infer()) + self.assertIsInstance(inferred, nodes.Const) + self.assertEqual(inferred.value, 42) + def _slicing_test_helper(self, pairs, cls, get_elts): for code, expected in pairs: ast_node = extract_node(code) @@ -3634,7 +3684,8 @@ class InferenceTest(resources.SysPathSetup, unittest.TestCase): flow = AttributeDict() flow['app'] = AttributeDict() flow['app']['config'] = AttributeDict() - flow['app']['config']['doffing'] = AttributeDict() #@ + flow['app']['config']['doffing'] = AttributeDict() + flow['app']['config']['doffing']['thinkto'] = AttributeDict() #@ """ ) self.assertIsNone(helpers.safe_infer(ast_node.targets[0])) @@ -3725,6 +3776,40 @@ class InferenceTest(resources.SysPathSetup, unittest.TestCase): self.assertIsInstance(inferred, nodes.ClassDef) self.assertEqual(inferred.name, "B") + def test_With_metaclass_subclasses_arguments_are_classes_not_instances(self): + ast_node = extract_node( + """ + class A(type): + def test(cls): + return cls + import six + class B(six.with_metaclass(A)): + pass + + B.test() #@ + """ + ) + inferred = next(ast_node.infer()) + self.assertIsInstance(inferred, nodes.ClassDef) + self.assertEqual(inferred.name, "B") + + def test_With_metaclass_with_partial_imported_name(self): + ast_node = extract_node( + """ + class A(type): + def test(cls): + return cls + from six import with_metaclass + class B(with_metaclass(A)): + pass + + B.test() #@ + """ + ) + inferred = next(ast_node.infer()) + self.assertIsInstance(inferred, nodes.ClassDef) + self.assertEqual(inferred.name, "B") + def test_infer_cls_in_class_methods(self): ast_nodes = extract_node( """ @@ -5854,5 +5939,19 @@ def test_infer_generated_setter(): assert list(inferred.nodes_of_class(nodes.Const)) == [] +def test_infer_list_of_uninferables_does_not_crash(): + code = """ + x = [A] * 1 + f = [x, [A] * 2] + x = list(f) + [] # List[Uninferable] + tuple(x[0]) + """ + node = extract_node(code) + inferred = next(node.infer()) + assert isinstance(inferred, nodes.Tuple) + # Would not be able to infer the first element. + assert not inferred.elts + + if __name__ == "__main__": unittest.main() diff --git a/tests/unittest_nodes.py b/tests/unittest_nodes.py index 89140116..3396f91e 100644 --- a/tests/unittest_nodes.py +++ b/tests/unittest_nodes.py @@ -1157,6 +1157,19 @@ def test_type_comments_posonly_arguments(): assert actual_arg.as_string() == expected_arg +@pytest.mark.skipif(not HAS_TYPED_AST, reason="requires typed_ast") +def test_correct_function_type_comment_parent(): + data = """ + def f(a): + # type: (A) -> A + pass + """ + astroid = builder.parse(data) + f = astroid.body[0] + assert f.type_comment_args[0].parent is f + assert f.type_comment_returns.parent is f + + def test_is_generator_for_yield_assignments(): node = astroid.extract_node( """ @@ -1334,5 +1347,16 @@ def test_is_generator_for_yield_in_if(): assert bool(node.is_generator()) +def test_is_generator_for_yield_in_aug_assign(): + code = """ + def test(): + buf = '' + while True: + buf += yield + """ + node = astroid.extract_node(code) + assert bool(node.is_generator()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unittest_regrtest.py b/tests/unittest_regrtest.py index 45fbabf8..582e5072 100644 --- a/tests/unittest_regrtest.py +++ b/tests/unittest_regrtest.py @@ -92,7 +92,7 @@ class NonRegressionTests(resources.AstroidCacheSetupMixin, unittest.TestCase): data = """ from numpy import multiply -multiply(1, 2, 3) +multiply([1, 2], [3, 4]) """ astroid = builder.string_build(data, __name__, __file__) callfunc = astroid.body[1].value.func |