Accept both 'int' and 'long' for 'x: int' annotations in Py2 (GH-5097)

Disregard 'x: int' type annotations only when language_level=2, and otherwise make sure that we accept both int and long in Py2.

Closes https://github.com/cython/cython/issues/4944

10 files changed, 75 insertions, 23 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 7a07e4880..fe9565d45 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -56,6 +56,11 @@ Bugs fixed
   on creation, as Python does, and not later on start, as they did previously.
   (Github issue :issue:`1159`)
 
+* Type annotations for Python ``int`` rejected ``long`` under Py2 in the alpha-11 release.
+  They are now ignored again (as always before) when ``language_level=2``, and accept
+  both ``int`` and ``long`` in Py2 (and only ``int`` in Py3) otherwise.
+  (Github issue :issue:`4944`)
+
 * Iterating over memoryviews in generator expressions could leak a buffer reference.
   (Github issue :issue:`4968`)
 
diff --git a/Cython/Compiler/Builtin.py b/Cython/Compiler/Builtin.py
index 26fd68ff6..5d9579c9a 100644
--- a/Cython/Compiler/Builtin.py
+++ b/Cython/Compiler/Builtin.py
@@ -441,11 +441,10 @@ def init_builtins():
     basestring_type = builtin_scope.lookup('basestring').type
     bytearray_type = builtin_scope.lookup('bytearray').type
     float_type = builtin_scope.lookup('float').type
+    int_type = builtin_scope.lookup('int').type
     long_type = builtin_scope.lookup('long').type
     bool_type  = builtin_scope.lookup('bool').type
     complex_type  = builtin_scope.lookup('complex').type
-    # Be careful with int type while Py2 is still supported
-    int_type = builtin_scope.lookup('int').type
 
     # Set up type inference links between equivalent Python/C types
     bool_type.equivalent_type = PyrexTypes.c_bint_type
diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py
index df95b8615..1029d835e 100644
--- a/Cython/Compiler/ExprNodes.py
+++ b/Cython/Compiler/ExprNodes.py
@@ -1535,13 +1535,16 @@ def _analyse_name_as_type(name, pos, env):
     if ctype is not None and env.in_c_type_context:
         return ctype
 
-    global_entry = env.global_scope().lookup(name)
+    global_scope = env.global_scope()
+    global_entry = global_scope.lookup(name)
     if global_entry and global_entry.is_type:
         type = global_entry.type
-        if (not env.in_c_type_context and
-                name == 'int' and type is Builtin.int_type):
+        if (not env.in_c_type_context
+                and type is Builtin.int_type
+                and global_scope.context.language_level == 2):
             # While we still support Python2 this needs to be downgraded
-            # to a generic Python object to include both int and long
+            # to a generic Python object to include both int and long.
+            # With language_level > 3, we keep the type but also accept 'long' in Py2.
             type = py_object_type
         if type and (type.is_pyobject or env.in_c_type_context):
             return type
@@ -2124,9 +2127,10 @@ class NameNode(AtomicExprNode):
                 type = py_object_type
             elif type.is_pyobject and type.equivalent_type:
                 type = type.equivalent_type
-            elif type is Builtin.int_type:
-                # while we still support Python 2 this must be an object
-                # so that it can be either int or long
+            elif type is Builtin.int_type and env.global_scope().context.language_level == 2:
+                # While we still support Python 2 this must be a plain object
+                # so that it can be either int or long.  With language_level=3(str),
+                # we pick up the type but accept both int and long in Py2.
                 type = py_object_type
             return type
         if self.name == 'object':
diff --git a/Cython/Compiler/Optimize.py b/Cython/Compiler/Optimize.py
index 231d23419..66386700e 100644
--- a/Cython/Compiler/Optimize.py
+++ b/Cython/Compiler/Optimize.py
@@ -2860,6 +2860,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
                     builtin_type = None
             if builtin_type is not None:
                 type_check_function = entry.type.type_check_function(exact=False)
+                if type_check_function == '__Pyx_Py3Int_Check' and builtin_type is Builtin.int_type:
+                    # isinstance(x, int) should really test for 'int' in Py2, not 'int | long'
+                    type_check_function = "PyInt_Check"
                 if type_check_function in tests:
                     continue
                 tests.append(type_check_function)
diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
index da30809a3..4061b3b0b 100644
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
@@ -1456,6 +1456,9 @@ class BuiltinObjectType(PyObjectType):
             type_check = 'PyByteArray_Check'
         elif type_name == 'frozenset':
             type_check = 'PyFrozenSet_Check'
+        elif type_name == 'int':
+            # For backwards compatibility of (Py3) 'x: int' annotations in Py2, we also allow 'long' there.
+            type_check = '__Pyx_Py3Int_Check'
         else:
             type_check = 'Py%s_Check' % type_name.capitalize()
         if exact and type_name not in ('bool', 'slice', 'Exception'):
diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
index df2a4ee4a..35083a6fe 100644
--- a/Cython/Utility/ModuleSetupCode.c
+++ b/Cython/Utility/ModuleSetupCode.c
@@ -1144,6 +1144,8 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict,
   #define PyInt_Type                   PyLong_Type
   #define PyInt_Check(op)              PyLong_Check(op)
   #define PyInt_CheckExact(op)         PyLong_CheckExact(op)
+  #define __Pyx_Py3Int_Check(op)       PyLong_Check(op)
+  #define __Pyx_Py3Int_CheckExact(op)  PyLong_CheckExact(op)
   #define PyInt_FromString             PyLong_FromString
   #define PyInt_FromUnicode            PyLong_FromUnicode
   #define PyInt_FromLong               PyLong_FromLong
@@ -1155,6 +1157,9 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict,
   #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask
   #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask
   #define PyNumber_Int                 PyNumber_Long
+#else
+  #define __Pyx_Py3Int_Check(op)       (PyLong_Check(op) || PyInt_Check(op))
+  #define __Pyx_Py3Int_CheckExact(op)  (PyLong_CheckExact(op) || PyInt_CheckExact(op))
 #endif
 
 #if PY_MAJOR_VERSION >= 3
diff --git a/docs/src/userguide/migrating_to_cy30.rst b/docs/src/userguide/migrating_to_cy30.rst
index bf0b7972a..f23cd4cfe 100644
--- a/docs/src/userguide/migrating_to_cy30.rst
+++ b/docs/src/userguide/migrating_to_cy30.rst
@@ -224,6 +224,12 @@ annotations and it is well worth reading
 :ref:`the pure Python tutorial<pep484_type_annotations>` to understand
 some of the improvements.
 
+A notable backwards-incompatible change is that ``x: int`` is now typed
+such that ``x`` is an exact Python ``int`` (Cython 0.29 would accept
+any Python object for ``x``), unless the language level is explicitly
+set to 2.  To mitigate the effect, Cython 3.0 still accepts both Python
+``int`` and ``long`` values under Python 2.x.
+
 To make it easier to handle cases where your interpretation of type
 annotations differs from Cython's, Cython 3 now supports setting the
 ``annotation_typing`` :ref:`directive <compiler-directives>` on a
diff --git a/docs/src/userguide/source_files_and_compilation.rst b/docs/src/userguide/source_files_and_compilation.rst
index 42e092d0a..0a1988442 100644
--- a/docs/src/userguide/source_files_and_compilation.rst
+++ b/docs/src/userguide/source_files_and_compilation.rst
@@ -905,13 +905,15 @@ Cython code.  Here is the list of currently supported directives:
     explicitly requested.
 
 ``language_level`` (2/3/3str)
-    Globally set the Python language level to be used for module
-    compilation.  Default is compatibility with Python 2.  To enable
-    Python 3 source code semantics, set this to 3 (or 3str) at the start
+    Globally set the Python language level to be used for module compilation.
+    Default is compatibility with Python 3 in Cython 3.x and with Python 2 in Cython 0.x.
+    To enable Python 3 source code semantics, set this to 3 (or 3str) at the start
     of a module or pass the "-3" or "--3str" command line options to the
-    compiler.  The ``3str`` option enables Python 3 semantics but does
+    compiler.  For Python 2 semantics, use 2 and "-2" accordingly.  The ``3str``
+    option enables Python 3 semantics but does
     not change the ``str`` type and unprefixed string literals to
     ``unicode`` when the compiled code runs in Python 2.x.
+    Language level 2 ignores ``x: int`` type annotations due to the int/long ambiguity.
     Note that cimported files inherit this setting from the module
     being compiled, unless they explicitly set their own language level.
     Included source files always inherit this setting.
diff --git a/tests/run/cython3.pyx b/tests/run/cython3.pyx
index 9dae55301..dcc250ae0 100644
--- a/tests/run/cython3.pyx
+++ b/tests/run/cython3.pyx
@@ -633,10 +633,12 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar
     return result
 
 
-def builtin_as_annotation(text: str):
+@cython.annotation_typing(False)
+def builtin_as_ignored_annotation(text: str):
+    # Used to crash the compiler when annotation typing is disabled.
     # See https://github.com/cython/cython/issues/2811
     """
-    >>> builtin_as_annotation("abc")
+    >>> builtin_as_ignored_annotation("abc")
     a
     b
     c
@@ -645,6 +647,24 @@ def builtin_as_annotation(text: str):
         print(c)
 
 
+@cython.annotation_typing(True)
+def int_annotation(x: int) -> int:
+    """
+    >>> print(int_annotation(1))
+    2
+    >>> print(int_annotation(10))
+    1024
+    >>> print(int_annotation(100))
+    1267650600228229401496703205376
+    >>> print(int_annotation((10 * 1000 * 1000) // 1000 // 1000))  # 'long' arg in Py2
+    1024
+    >>> print(int_annotation((100 * 1000 * 1000) // 1000 // 1000))  # 'long' arg in Py2
+    1267650600228229401496703205376
+    """
+    return 2 ** x
+
+
+@cython.annotation_typing(True)
 async def async_def_annotations(x: 'int') -> 'float':
     """
     >>> ret, arg = sorted(async_def_annotations.__annotations__.items())
@@ -656,11 +676,3 @@ async def async_def_annotations(x: 'int') -> 'float':
     'int'
     """
     return float(x)
-
-
-def repr_returns_str(x) -> str:
-    """
-    >>> repr_returns_str(123)
-    '123'
-    """
-    return repr(x)
diff --git a/tests/run/cython3_no_unicode_literals.pyx b/tests/run/cython3_no_unicode_literals.pyx
index 99e8b3f1a..308de75a5 100644
--- a/tests/run/cython3_no_unicode_literals.pyx
+++ b/tests/run/cython3_no_unicode_literals.pyx
@@ -4,6 +4,8 @@
 
 print(end='')  # test that language_level 3 applies immediately at the module start, for the first token.
 
+import cython
+
 __doc__ = """
 >>> items = sorted(locals_function(1).items())
 >>> for item in items:
@@ -147,6 +149,8 @@ def strip_wrapped_string(s):
     assert s[0] == s[-1] # delimiters on either end are the same
     return s[1:-1] # strip them
 
+
+@cython.annotation_typing(False)
 def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwargs: "KWARGS") -> "ret":
     """
     >>> annotation_syntax(1)
@@ -170,3 +174,12 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar
     result : int = a + b
 
     return result
+
+
+@cython.annotation_typing(True)
+def repr_returns_str(x) -> str:
+    """
+    >>> repr_returns_str(123)
+    '123'
+    """
+    return repr(x)